Initial checkin of Keras Optimzers and helper classes. (tensorflow#91)

* Initial checkin of Keras Optimzers and helper classes.
Fixed dependencies in pom.xml

* Added static final NAME to replace hardcoded String in the create method. This allows the NAME to be used elsewhere instead of hardcoding the string.

* Changed of method to use the DataType NAME attribute rather than hardcoding the string.
added methods isFloating(), isInteger(), isNUmeric(), isBoolean() and isString()

* Added method WriteFieldWithInitializer to output a "final static String OP_NAME" to each generated operation.

* Added tf.nn.softmaxCrossEntropyWitLogits() and tf.nn.raw.softmaxCrossEntropyWitLogits()
Added tf.nn.sparesSoftmaxCrossEntropyWithLogits() and
tf.nn.raw.sparesSoftmaxCrossEntropyWithLogits()

Added tf.nn.sigmoidCrossEntropyWithLogits()

* Moved SoftmaxCrossEntropyWithLogits and  SparseSoftmaxCrossEntropyWithLogits to org.tensorflow.op.nn.raw

* Generated classes now have public static final String OP_NAME = "XXXXXXXX";

* Generated classes now have public static final String OP_NAME = "XXXXXXXX";

* fix dependencies for other Tensorflow Java modules

* formatting fix

* Fix ctors with name to properly pass the name to the the super ctor.

* change asserts to IllegalArgumentException
fix javadoc, fix casts

* change asserts to IllegalArgumentException

* Moved back to tests

* Moved SoftmaxCrossEntropyWithLogits.java and SparseSoftmaxCrossEntropyWithLogits.java to nn.raw,
added new versions of these to NnOps

* Deleted files that are not necessary yet

* Added nn.raw group for softmaxCrossEntropyWithLogits() and sparseSoftmaxCrossEntropyWithLogits()

* Added nn.raw group for softmaxCrossEntropyWithLogits() and sparseSoftmaxCrossEntropyWithLogits()

* Refactor NN into individual operations under org.tensorflow.op.nn. Fix JavaDoc. Change from snake case to camel case.

* Refactor NN into individual operations under org.tensorflow.op.nn. Fix JavaDoc. Change from snake case to camel case.

* Reformatted code

* Added sub scope

* Miscellaneous fixes based on review comments.

* Fixed op_generator.cc to remove a spurious new line in the generated Java files for some Ops. This also  resulted in new generated  source that are also committed.

* Changed back to non-generic Operand until we resolve how to handle generics.

* Regenerated due to creation of SoftmaxCrossEntropyWithLogits.java,  SigmoidCrossEntropyWithLogits.java, and SparseSoftmaxCrossEntropyWithLogits.java under package org.tensorflow.op.nn in

* change snake case to camel case. format code

* clean upd warning,  format code

* Added Adamax, Ftrl, and Nadam Optimizers. Added Optimizers enum for easy inclusion of a default optimizer. Cleaned up JavaDoc

* Removed optimize classes from tensorflow-keras, moved optimizer test cases to framework. Created Tests for GradientDescent and Momentum

* Fixed generics

* Fixed from Unit test results

* added @SuppressWarnings("unchecked") on Variable array

Author: JimClarke5

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_SoftmaxCrossEntropyWithLogits.pbtxt

@@ -1,6 +1,6 @@
 op {
   graph_op_name: "SoftmaxCrossEntropyWithLogits"
   endpoint {
-    name: "nn.SoftmaxCrossEntropyWithLogits"
+    name: "nn.raw.SoftmaxCrossEntropyWithLogits"
   }
 }

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_SparseSoftmaxCrossEntropyWithLogits.pbtxt

@@ -1,6 +1,6 @@
 op {
   graph_op_name: "SparseSoftmaxCrossEntropyWithLogits"
   endpoint {
-    name: "nn.SparseSoftmaxCrossEntropyWithLogits"
+    name: "nn.raw.SparseSoftmaxCrossEntropyWithLogits"
   }
 }

tensorflow-core/tensorflow-core-api/src/bazel/op_generator/op_generator.cc

@@ -509,9 +509,20 @@ void GenerateOp(const OpSpec& op, const EndpointSpec& endpoint,
     RenderInterfaceImpl(op, mode, &writer);
   }
   writer.EndLine();
-  for (const ArgumentSpec& output : op.outputs()) {
-    writer.WriteField(output.var(), PRIVATE);
+
+  Variable nameVariable = Variable::Create("OP_NAME", Type::Class("String")); 
+  Javadoc name_javadoc = Javadoc::Create("The name of this op, as known by TensorFlow core engine");
+  string quoted_string = "\"" + op.graph_op_name() + "\"";
+  writer.WriteFieldWithInitializer(nameVariable, PUBLIC|STATIC|FINAL, &name_javadoc, quoted_string );
+
+
+  if(!op.outputs().empty()) {
+    writer.EndLine();
+    for (const ArgumentSpec& output : op.outputs()) {
+        writer.WriteField(output.var(), PRIVATE);
+    }
   }
+
   RenderConstructor(op, op_class, &writer);
   writer.EndType();
 }

tensorflow-core/tensorflow-core-api/src/bazel/op_generator/source_writer.cc

@@ -238,6 +238,22 @@ SourceWriter& SourceWriter::WriteField(const Variable& field, int modifiers,
   return *this;
 }
 
+SourceWriter& SourceWriter::WriteFieldWithInitializer(const Variable& field,
+	int modifiers, const Javadoc* javadoc, const string& initializer) {
+  // If present, write field javadoc only as one brief line
+  if (javadoc != nullptr && !javadoc->brief().empty()) {
+    Append("/** ").Append(javadoc->brief()).Append(" */").EndLine();
+  }
+  WriteModifiers(modifiers);
+  if (!initializer.empty())
+    AppendType(field.type()).Append(" ").Append(field.name()).
+                            Append(" = ").Append(initializer).Append(";");
+  else
+    AppendType(field.type()).Append(" ").Append(field.name()).Append(";");
+  EndLine();
+  return *this;
+}
+
 SourceWriter& SourceWriter::WriteModifiers(int modifiers) {
   if (modifiers & PUBLIC) {
     Append("public ");

tensorflow-core/tensorflow-core-api/src/bazel/op_generator/source_writer.h

@@ -153,6 +153,9 @@ class SourceWriter {
   SourceWriter& WriteField(const Variable& field, int modifiers,
                            const Javadoc* javadoc = nullptr);
 
+  SourceWriter& WriteFieldWithInitializer(const Variable& field,
+        int modifiers, const Javadoc* javadoc = nullptr, const string& initializer = nullptr);
+
  protected:
   virtual void DoAppend(const StringPiece& str) = 0;
 
@@ -161,7 +164,7 @@ class SourceWriter {
   class TypeVisitor {
    public:
     virtual ~TypeVisitor() = default;
-    void Visit(const Type& type);
+    void Visit(const Type& type);	
 
    protected:
     virtual void DoVisit(const Type& type) = 0;

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnOps.java

@@ -83,6 +83,7 @@
 import org.tensorflow.op.nn.Relu;
 import org.tensorflow.op.nn.Relu6;
 import org.tensorflow.op.nn.Selu;
+import org.tensorflow.op.nn.SigmoidCrossEntropyWithLogits;
 import org.tensorflow.op.nn.Softmax;
 import org.tensorflow.op.nn.SoftmaxCrossEntropyWithLogits;
 import org.tensorflow.op.nn.Softsign;
@@ -102,10 +103,13 @@
  * @see {@link Ops}
  */
 public final class NnOps {
+  public final NnRawOps raw;
+
   private final Scope scope;
 
   NnOps(Scope scope) {
     this.scope = scope;
+    raw = new NnRawOps(scope);
   }
 
   /**
@@ -1753,6 +1757,56 @@ public <T extends TNumber> Selu<T> selu(Operand<T> features) {
     return Selu.create(scope, features);
   }
 
+  /**
+   * Computes sigmoid cross entropy given <code>logits</code>.
+   *
+   *  <p>Measures the probability error in discrete classification tasks in which each class is
+   *  independent and not mutually exclusive. For instance, one could perform multilabel
+   *  classification where a picture can contain both an elephant and a dog at the same time.
+   *
+   *  <p>For brevity, let <code>x = logits</code>, <code>z = labels</code>. The logistic loss in
+   *  pseudo-code is
+   *
+   *  <pre>
+   *  z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
+   *   = z * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))
+   *   = z * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))
+   *   = z * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))
+   *   = (1 - z) * x + log(1 + exp(-x))
+   *   = x - x * z + log(1 + exp(-x))
+   *  </pre>
+   *
+   *  <p>For <code>x < 0</code>, to avoid overflow in <code>exp(-x)</code>, we reformulate the above
+   *
+   *  <pre>
+   *  x - x * z + log(1 + exp(-x))
+   *   = log(exp(x)) - x * z + log(1 + exp(-x))
+   *   = - x * z + log(1 + exp(x))
+   *  </pre>
+   *
+   *  <p>Hence, to ensure stability and avoid overflow, the implementation uses this equivalent
+   *  formulation
+   *
+   *  <pre>
+   *    max(x, 0) - x * z + log(1 + exp(-abs(x)))
+   *  </pre>
+   *
+   *  <p></ode>logits</code> and <code>labels</code> must have the same type and shape.
+   *
+   *  <p>
+   *
+   * @param scope The TensorFlow scope
+   * @param labels the labels
+   * @param logits the logits of type float32 or float64
+   * @param <T> the type of labels and logits
+   * @return the component-wise logistic losses.
+   * @throws IllegalArgumentException if logits' and labels' do not have the same shape
+   */
+  public <T extends TNumber> Operand<T> sigmoidCrossEntropyWithLogits(Operand<T> labels,
+      Operand<T> logits) {
+    return SigmoidCrossEntropyWithLogits.sigmoidCrossEntropyWithLogits(scope, labels, logits);
+  }
+
   /**
    * Computes softmax activations.
    *  <p>
@@ -1769,20 +1823,54 @@ public <T extends TNumber> Softmax<T> softmax(Operand<T> logits) {
   }
 
   /**
-   * Computes softmax cross entropy cost and gradients to backpropagate.
-   *  <p>
-   *  Inputs are the logits, not probabilities.
+   * Computes softmax cross entropy between <code>logits</code> and <code>labels</code>.
    *
-   * @param <T> data type for {@code loss()} output
-   * @param features batch_size x num_classes matrix
-   * @param labels batch_size x num_classes matrix
-   *  The caller must ensure that each batch of labels represents a valid
-   *  probability distribution.
-   * @return a new instance of SoftmaxCrossEntropyWithLogits
+   *  <p>Measures the probability error in discrete classification tasks in which the classes are
+   *  mutually exclusive (each entry is in exactly one class). For example, each CIFAR-10 image is
+   *  labeled with one and only one label: an image can be a dog or a truck, but not both.
+   *
+   *  <p><b>NOTE:</b>
+   *
+   *  <p>While the classes are mutually exclusive, their probabilities need not be. All that is
+   *  required is that each row of <code>labels</code> is a valid probability distribution. If they
+   *  are not, the computation of the gradient will be incorrect.
+   *
+   *  <p>If using exclusive <code>labels</code> (wherein one and only one class is true at a time),
+   *  see {@link org.tensorflow.op.NnOps#sparseSoftmaxCrossEntropyWithLogits}
+   *
+   *  <p>Usage:
+   *
+   *  <pre>
+   *    Operand&lt;TFloat32&gt; logits =
+   *        tf.constant(new float[][] {{4.0F, 2.0F, 1.0F}, {0.0F, 5.0F, 1.0F}} );
+   *    Operand&lt;TFloat32&gt; labels =
+   *        tf.constant(new float[][] {{1.0F, 0.0F, 0.0F}, {0.0F, 0.8F, 0.2F}} );
+   *    Operand&lt;TFloat32&gt; output =
+   *        tf.nn.softmaxCrossEntropyWithLogits(labels, logits, -1);
+   *    // output Shape = [2]
+   *    // dataType = FLOAT (1)
+   *    // values { 0.169846, 0.824745 }
+   *  </pre>
+   *
+   *  <p>Backpropagation will happen into both <code>logits</code> and <code>labels</code>. To
+   *  disallow backpropagation into <code>labels</code>, pass label tensors through <code>
+   *  tf.stopGradient</code> before feeding it to this function.
+   *
+   * @param scope current scope
+   * @param labels Each vector along the class dimension should hold a valid probability
+   *      distribution e.g. for the case in which labels are of shape <code>[batch_size, num_classes]
+   *      </code>, each row of <code>labels[i]</code> must be a valid probability distribution.
+   * @param logits Per-label activations, typically a linear output. These activation energies are
+   *      interpreted as unnormalized log probabilities.
+   * @param axis The class dimension. -1 is the last dimension.
+   * @param <T> the number type of the operands
+   * @return the softmax cross entropy loss. Its type is the same as <code>logits</code> and its
+   *      shape is the same as <code>labels</code> except that it does not have the last dimension of
+   *      <code>labels</code>.
    */
-  public <T extends TNumber> SoftmaxCrossEntropyWithLogits<T> softmaxCrossEntropyWithLogits(
-      Operand<T> features, Operand<T> labels) {
-    return SoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  public <T extends TNumber, U extends TNumber> Operand<T> softmaxCrossEntropyWithLogits(
+      Operand<U> labels, Operand<T> logits, int axis) {
+    return SoftmaxCrossEntropyWithLogits.softmaxCrossEntropyWithLogits(scope, labels, logits, axis);
   }
 
   /**
@@ -1974,24 +2062,51 @@ public <T extends TType> SpaceToDepth<T> spaceToDepth(Operand<T> input, Long blo
   }
 
   /**
-   * Computes softmax cross entropy cost and gradients to backpropagate.
-   *  <p>
-   *  Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
-   *  a matrix of label probabilities, but rather a single label per row
-   *  of features.  This label is considered to have probability 1.0 for the
-   *  given row.
-   *  <p>
-   *  Inputs are the logits, not probabilities.
-   *
-   * @param <T> data type for {@code loss()} output
-   * @param features batch_size x num_classes matrix
-   * @param labels batch_size vector with values in [0, num_classes).
-   *  This is the label for the given minibatch entry.
-   * @return a new instance of SparseSoftmaxCrossEntropyWithLogits
-   */
-  public <T extends TNumber, U extends TNumber> SparseSoftmaxCrossEntropyWithLogits<T> sparseSoftmaxCrossEntropyWithLogits(
-      Operand<T> features, Operand<U> labels) {
-    return SparseSoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+   * Computes sparse softmax cross entropy between <code>logits</code> and <code>labels</code>.
+   *
+   *  <p>Measures the probability error in discrete classification tasks in which the classes are
+   *  mutually exclusive (each entry is in exactly one class). For example, each CIFAR-10 image is
+   *  labeled with one and only one label: an image can be a dog or a truck, but not both.
+   *
+   *  <p><b>NOTE:</b>
+   *
+   *  <p>For this operation, the probability of a given label is considered exclusive. That is, soft
+   *  classes are not allowed, and the <code>labels</code> vector must provide a single specific
+   *  index for the true class for each row of <code>logits</code> (each minibatch entry). For soft
+   *  softmax classification with a probability distribution for each entry, {@link
+   *  org.tensorflow.op.NnOps#softmaxCrossEntropyWithLogits}.
+   *
+   *  <p><b>WARNING:</b>
+   *
+   *  <p>This op expects unscaled logits, since it performs a <code>softmax</code> on <code>logits
+   *  </code> internally for efficiency. Do not call this op with the output of <code>softmax</code>,
+   *  as it will produce incorrect results.
+   *
+   *  <p>A common use case is to have logits of shape <code>[batchSize, numClasses]</code> and have
+   *  labels of shape <code>[batchSize]</code>, but higher dimensions are supported, in which case
+   *  the <code>dim</code>-th dimension is assumed to be of size <code>numClasses</code>. <code>
+   *  logits</code> must have the <cod>dataType</cod> of <code>TFloat16</code>, <code>TFloat32</code>
+   *  , or <code>TFloat64</code>, and <code>labels</code> must have the dtype of <code>TInt32</code>
+   *  or <code>TInt64</code>.
+   *
+   * @param scope current scope
+   * @param labels <code>Tensor</code> of shape <code>[d_0, d_1, ..., d_{r-1}]</code> (where <code>r
+   *      </code> is rank of <code>labels</code> and result) and the dataType is <code>TInt32</code>
+   *      or <code>TInt64</code>. Each entry in <code>labels</code> must be an index in <code>[0,
+   *      numClasses)</code>. Other values will raise an exception when this op is run on CPU, and
+   *      return <code>NaN</code> for corresponding loss and gradient rows on GPU.
+   * @param logits Per-label activations (typically a linear output) of shape <code>[d_0, d_1, ...,
+   *      d_{r-1}, numClasses]</code> and dataType of <code>TFloat16</code>, <code>TFloat32</code>,
+   *      or <code>TFloat64</code>. These activation energies are interpreted as unnormalized log
+   *      probabilities.
+   * @return A <code>Tensor</code> of the same shape as <code>labels</code> and of the same type as
+   *      <code>logits</code> with the softmax cross entropy loss.
+   * @throws IllegalArgumentException If logits are scalars (need to have rank >= 1) or if the rank
+   *      of the labels is not equal to the rank of the logits minus one.
+   */
+  public <T extends TNumber, U extends TNumber> Operand sparseSoftmaxCrossEntropyWithLogits(
+      Operand<T> labels, Operand<U> logits) {
+    return SparseSoftmaxCrossEntropyWithLogits.sparseSoftmaxCrossEntropyWithLogits(scope, labels, logits);
   }
 
   /**

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnRawOps.java

@@ -0,0 +1,74 @@
+// Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ==============================================================================
+//
+// This class has been generated, DO NOT EDIT!
+//
+package org.tensorflow.op;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.nn.raw.SoftmaxCrossEntropyWithLogits;
+import org.tensorflow.op.nn.raw.SparseSoftmaxCrossEntropyWithLogits;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * An API for building {@code nn.raw} operations as {@link Op Op}s
+ *
+ * @see {@link Ops}
+ */
+public final class NnRawOps {
+  private final Scope scope;
+
+  NnRawOps(Scope scope) {
+    this.scope = scope;
+  }
+
+  /**
+   * Computes softmax cross entropy cost and gradients to backpropagate.
+   *  <p>
+   *  Inputs are the logits, not probabilities.
+   *
+   * @param <T> data type for {@code loss()} output
+   * @param features batch_size x num_classes matrix
+   * @param labels batch_size x num_classes matrix
+   *  The caller must ensure that each batch of labels represents a valid
+   *  probability distribution.
+   * @return a new instance of SoftmaxCrossEntropyWithLogits
+   */
+  public <T extends TNumber> SoftmaxCrossEntropyWithLogits<T> softmaxCrossEntropyWithLogits(
+      Operand<T> features, Operand<T> labels) {
+    return SoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  }
+
+  /**
+   * Computes softmax cross entropy cost and gradients to backpropagate.
+   *  <p>
+   *  Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
+   *  a matrix of label probabilities, but rather a single label per row
+   *  of features.  This label is considered to have probability 1.0 for the
+   *  given row.
+   *  <p>
+   *  Inputs are the logits, not probabilities.
+   *
+   * @param <T> data type for {@code loss()} output
+   * @param features batch_size x num_classes matrix
+   * @param labels batch_size vector with values in [0, num_classes).
+   *  This is the label for the given minibatch entry.
+   * @return a new instance of SparseSoftmaxCrossEntropyWithLogits
+   */
+  public <T extends TNumber, U extends TNumber> SparseSoftmaxCrossEntropyWithLogits<T> sparseSoftmaxCrossEntropyWithLogits(
+      Operand<T> features, Operand<U> labels) {
+    return SparseSoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  }
+}

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/audio/AudioSpectrogram.java

@@ -127,6 +127,9 @@ public Output<TFloat32> asOutput() {
     return spectrogram;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "AudioSpectrogram";
+  
   private Output<TFloat32> spectrogram;
 
   private AudioSpectrogram(Operation operation) {

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/audio/DecodeWav.java

@@ -132,6 +132,9 @@ public Output<TInt32> sampleRate() {
     return sampleRate;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "DecodeWav";
+  
   private Output<TFloat32> audio;
   private Output<TInt32> sampleRate;