Optimizer package for Graph mode

.gitignore

@@ -37,6 +37,10 @@ xcuserdata/**
 /estimator_api_init_files_list.txt
 *.whl
 
+# Patch files
+*.orig
+*.rej
+
 # Android
 .gradle
 .idea

pom.xml

@@ -31,6 +31,7 @@
   <modules>
     <module>tensorflow-tools</module>
     <module>tensorflow-core</module>
+    <module>tensorflow-training</module>
   </modules>
 
   <properties>
@@ -39,6 +40,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <junit.version>4.12</junit.version>
     <jmh.version>1.21</jmh.version>
+    <skipAssembly>true</skipAssembly>
   </properties>
 
   <dependencyManagement>
@@ -120,6 +122,15 @@
         </executions>
       </plugin-->
     </plugins>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-jar-plugin</artifactId>
+          <version>3.2.0</version>
+        </plugin>
+      </plugins>
+    </pluginManagement>
   </build>
 
 </project>

tensorflow-core/tensorflow-core-api/pom.xml

@@ -332,6 +332,15 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>3.2.0</version>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/Ops.java

@@ -79,6 +79,7 @@
 import org.tensorflow.op.core.Gradients;
 import org.tensorflow.op.core.GuaranteeConst;
 import org.tensorflow.op.core.HashTable;
+import org.tensorflow.op.core.Helpers;
 import org.tensorflow.op.core.HistogramFixedWidth;
 import org.tensorflow.op.core.Identity;
 import org.tensorflow.op.core.IdentityN;
@@ -1846,7 +1847,7 @@ public Gradients gradients(Iterable<? extends Operand<?>> y, Iterable<? extends
    *  Example of usage:
    *  <pre>{@code
    *  Gradients gradients = tf.gradients(loss, Arrays.asList(w, b));
-   *  Scalar<TFloat32> alpha = ops.scalar(1.0f);
+   *  Constant<TFloat32> alpha = tf.val(1.0f);
    *  tf.train.applyGradientDescent(w, alpha, gradients.<Float>dy(0));
    *  tf.train.applyGradientDescent(b, alpha, gradients.<Float>dy(1));
    *  }</pre>
@@ -7332,6 +7333,21 @@ public VarIsInitializedOp varIsInitializedOp(Operand<?> resource) {
     return VarIsInitializedOp.create(scope, resource);
   }
 
+  /**
+   * Factory method to create a new Variable with its initializer.
+   *  <p>
+   *  Only supported on Graph sessions as the {@link org.tensorflow.op.core.Assign} op
+   *  does not work in an EagerSession.
+   *
+   * @param scope current scope
+   * @param init The op to use to initialise this variable.
+   * @param options carries optional attributes values
+   * @return a new instance of Variable
+   */
+  public <T extends TType> Variable<T> variable(Operand<T> init, Variable.Options... options) {
+    return Helpers.createVariableWithInit(scope, init, options);
+  }
+
   /**
    * Holds state in the form of a tensor that persists across steps.
    *  <p>

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/DataTypes.java

@@ -70,5 +70,6 @@ static DataType<?> fromNativeCode(int nativeCode) {
   //      to allow user to register custom data types?
   private static void register(DataType<?> dataType) {
     DATA_TYPE_REGISTRY.put(dataType.nativeCode(), dataType);
+    DATA_TYPE_REGISTRY.put(dataType.nativeCode() + 100, dataType);
   }
 }

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Graph.java

@@ -26,7 +26,9 @@
 import static org.tensorflow.internal.c_api.global.tensorflow.TF_NewGraph;
 import static org.tensorflow.internal.c_api.global.tensorflow.TF_NewWhile;
 
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import org.bytedeco.javacpp.BytePointer;
 import org.bytedeco.javacpp.Pointer;
 import org.bytedeco.javacpp.PointerScope;
@@ -38,6 +40,9 @@
 import org.tensorflow.internal.c_api.TF_Output;
 import org.tensorflow.internal.c_api.TF_Status;
 import org.tensorflow.internal.c_api.TF_WhileParams;
+import org.tensorflow.op.Scope;
+import org.tensorflow.op.core.NoOp;
+
 
 /**
  * A data flow graph representing a TensorFlow computation.
@@ -49,6 +54,8 @@
  */
 public final class Graph implements ExecutionEnvironment, AutoCloseable {
 
+  public static final String DEFAULT_INIT_NAME = "init";
+
   /** Create an empty Graph. */
   public Graph() {
     nativeHandle = allocate();
@@ -166,6 +173,28 @@ public byte[] toGraphDef() {
     }
   }
 
+  /**
+   * Adds an initializer to the graph initializer list.
+   * @param initializer An initializer to add to the list.
+   */
+  public synchronized void addInitializer(Operand<?> initializer) {
+    initializers.add(initializer);
+  }
+
+  /**
+   * Returns an op which initializers all the variables.
+   * @return The initializer operation.
+   */
+  public NoOp variablesInitializer() {
+    return variablesInitializer(DEFAULT_INIT_NAME);
+  }
+
+  public NoOp variablesInitializer(String name) {
+    Scope scope = new Scope(this);
+    scope = scope.withName(name).withControlDependencies(initializers);
+    return NoOp.create(scope);
+  }
+
   /**
    * Adds operations to compute the partial derivatives of sum of {@code y}s w.r.t {@code x}s, i.e.,
    * {@code d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...}
@@ -378,6 +407,8 @@ public Output<?>[] whileLoop(
   private TF_Graph nativeHandle;
   private int refcount = 0;
 
+  private final List<Operand<?>> initializers = new ArrayList<>();
+
   // Related native objects (such as the TF_Operation object backing an Operation instance)
   // have a validity tied to that of the Graph. The handles to those native objects are not
   // valid after Graph.close() has been invoked.

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java

@@ -146,7 +146,6 @@ public final class Tensor<T extends TType> implements AutoCloseable {
    * @throws IllegalArgumentException if {@code obj} is not compatible with the TensorFlow type
    *     system.
    */
-  @SuppressWarnings("unchecked")
   public static <T extends TType> Tensor<T> create(Object obj, DataType<T> dtype) {
     if (!objectCompatWithType(obj, dtype)) {
       throw new IllegalArgumentException(
@@ -158,7 +157,7 @@ public static <T extends TType> Tensor<T> create(Object obj, DataType<T> dtype)
     }
     long[] dimSizes = new long[numDimensions(obj, dtype)];
     fillShape(obj, 0, dimSizes);
-    Tensor<T> t = new Tensor(dtype, Shape.of(dimSizes));
+    Tensor<T> t = new Tensor<>(dtype, Shape.of(dimSizes));
     TF_Tensor nativeHandle;
     if (t.dtype != TString.DTYPE) {
       long byteSize = elemByteSize(t.dtype) * t.shape.size();

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/op/core/Helpers.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.tensorflow.op.core;
+
+import org.tensorflow.ExecutionEnvironment;
+import org.tensorflow.Graph;
+import org.tensorflow.Operand;
+import org.tensorflow.Output;
+import org.tensorflow.op.Scope;
+import org.tensorflow.op.annotation.Endpoint;
+import org.tensorflow.op.annotation.Operator;
+import org.tensorflow.types.family.TType;
+
+/**
+ * Container class for core methods which add or perform several operations
+ * and return one of them.
+ */
+@Operator
+public abstract class Helpers {
+
+    /**
+     * This class contains static factories.
+     */
+    private Helpers() {}
+
+    /**
+     * Factory method to create a new Variable with it's initializer.
+     * <p>
+     * Only supported on Graph sessions as the {@link org.tensorflow.op.core.Assign} op
+     * does not work in an EagerSession.
+     * @param scope current scope
+     * @param init The op to use to initialise this variable.
+     * @param options carries optional attributes values
+     * @return a new instance of Variable
+     */
+    @Endpoint(name="variable")
+    public static <T extends TType> Variable<T> createVariableWithInit(Scope scope, Operand<T> init, Variable.Options... options) {
+        Output<T> initOutput = init.asOutput();
+        Variable<T> newVar = Variable.create(scope,initOutput.shape(),initOutput.dataType(),options);
+        Assign<T> assignOp = Assign.create(scope,newVar,init);
+        ExecutionEnvironment exEnv = scope.env();
+        if (exEnv instanceof Graph) {
+            Graph graph = (Graph) exEnv;
+            graph.addInitializer(assignOp);
+        } else {
+            throw new IllegalArgumentException("variable with init is only supported on Graph sessions.");
+        }
+
+        return newVar;
+    }
+}

tensorflow-tools/src/main/java/org/tensorflow/tools/ndarray/NdArrays.java

@@ -456,8 +456,8 @@ public static <T> NdArray<T> scalarOfObject(T value) {
    */
   @SafeVarargs
   public static <T> NdArray<T> vectorOfObjects(T... values) {
-    if (values == null) {
-      throw new IllegalArgumentException();
+    if (values == null || values.length == 0) {
+      throw new IllegalArgumentException("Null or zero length input supplied to vectorOfObjects.");
     }
     return wrap(Shape.of(values.length), DataBuffers.from(values, false, false));
   }

tensorflow-training/pom.xml

@@ -0,0 +1,77 @@
+<!--
+  Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  =======================================================================
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.tensorflow</groupId>
+    <artifactId>tensorflow-java</artifactId>
+    <version>0.1.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>tensorflow-training</artifactId>
+  <packaging>jar</packaging>
+
+  <name>TensorFlow Training Library</name>
+  <description>
+    Operations for training Tensorflow models.
+  </description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.tensorflow</groupId>
+      <artifactId>tensorflow-core-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.22.2</version>
+        <configuration>
+          <forkCount>1</forkCount>
+          <reuseForks>false</reuseForks>
+          <argLine>-Xmx2G -XX:MaxPermSize=256m</argLine>
+          <skipTests>false</skipTests>
+          <includes>
+            <include>**/*Test.java</include>
+          </includes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>