Framework ops

.github/workflows/ci.yml

@@ -40,6 +40,32 @@ jobs:
       - name: Run lint checks
         run: |
           mvn compiler:compile -Pdev,jdk11 -B -U -e
+  check-format:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    container: centos:7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+      - name: Install environment
+        run: |
+          yum -y update
+          yum -y install centos-release-scl-rh epel-release
+          yum -y install java-11-openjdk-devel devtoolset-7
+          echo Downloading Maven
+          curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz
+          tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/
+          ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
+      - name: Build project
+        run: |
+          source scl_source enable devtoolset-7 || true
+          export JAVA_HOME=$(dirname $(dirname $(readlink $(readlink $(which javac)))))
+          echo $JAVA_HOME
+          mvn -version
+          mvn clean install -Pdev,jdk11 -B -U -e -Dlint.skip=true -Dmaven.test.skip=true
+      - name: Run format checks
+        run: |
+          mvn spotless:check -Pdev,jdk11 -B -U -e
   prepare:
     runs-on: ubuntu-latest
     outputs:
@@ -63,7 +89,7 @@ jobs:
     needs: prepare
     strategy:
       matrix:
-        ext: ["", -mkl, -gpu, -mkl-gpu]
+        ext: ["", -gpu] #, -mkl, -mkl-gpu]
     steps:
       - name: Install environment
         run: |
@@ -78,7 +104,7 @@ jobs:
           tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/
           ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
           echo Downloading Bazel
-          curl -L https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh -o bazel.sh --retry 10
+          curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-linux-x86_64.sh -o bazel.sh --retry 10
           bash bazel.sh
           if [[ "${{ matrix.ext }}" == *-gpu ]]; then
           echo Installing CUDA
@@ -127,13 +153,13 @@ jobs:
     needs: prepare
     strategy:
       matrix:
-        ext: ["", -mkl]
+        ext: [""] # , -mkl]
     steps:
       - name: Install environment
         run: |
           python3 -m pip install numpy six
           echo Downloading Bazel
-          curl -L https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-darwin-x86_64.sh -o bazel.sh --retry 10
+          curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-darwin-x86_64.sh -o bazel.sh --retry 10
           bash bazel.sh
           brew install libomp perl
       - name: Checkout repository
@@ -163,7 +189,7 @@ jobs:
     needs: prepare
     strategy:
        matrix:
-         ext: ["", -gpu, -mkl] #, -mkl-gpu]
+         ext: ["", -gpu] #, -mkl, -mkl-gpu]
     steps:
       - name: Configure page file
         uses: al-cheb/[email protected]
@@ -182,7 +208,7 @@ jobs:
           bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;"
           echo Downloading Bazel
           mkdir C:\bazel
-          curl.exe -L https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
+          curl.exe -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
           set "EXT=${{ matrix.ext }}"
           if "%EXT:~-4%" == "-gpu" (
             echo Removing some unused stuff to avoid running out of disk space

CONTRIBUTING.md

@@ -21,6 +21,20 @@ the `dev` profile in your Maven command to use those artifacts instead of buildi
 Modifying the native op generation code (not the annotation processor) or the JavaCPP configuration (not the abstract Pointers) will require a
 complete build could be required to reflect the changes, otherwise `-Pdev` should be fine.
 
+## JDK 16+
+
+If you're using JDK 16+, you need to add some exports for the formatter plugin:
+
+```
+--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+```
+
+This can be done in `.mvn/jvm.config` or `MAVEN_OPTS`.
+
 ### Native Builds
 
 In some cases, like when adding GPU support or re-generating op classes, you will need to re-build the native library. 99% of this is building

README.md

@@ -146,7 +146,7 @@ This table shows the mapping between different version of TensorFlow for Java an
 | 0.2.0  | 2.3.1  |
 | 0.3.0  | 2.4.1  |
 | 0.3.1  | 2.4.1  |
-| 0.4.0-SNAPSHOT | 2.4.1
+| 0.4.0-SNAPSHOT | 2.5.0
 
 ## How to Contribute?
 

pom.xml

@@ -1,5 +1,5 @@
 <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
 
   <modelVersion>4.0.0</modelVersion>
 
@@ -44,6 +44,7 @@
     <maven.javadoc.skip>true</maven.javadoc.skip>
     <maven.source.skip>true</maven.source.skip>
     <gpg.skip>true</gpg.skip>
+    <spotless.version>2.11.1</spotless.version>
   </properties>
 
   <repositories>
@@ -102,19 +103,19 @@
         <artifactId>junit-jupiter-engine</artifactId>
         <version>${junit.version}</version>
         <scope>test</scope>
-    </dependency>
-	  <dependency>
-	    <groupId>org.openjdk.jmh</groupId>
-	    <artifactId>jmh-core</artifactId>
-	    <version>${jmh.version}</version>
-  	    <scope>test</scope>
-	  </dependency>
-	  <dependency>
-	    <groupId>org.openjdk.jmh</groupId>
-	    <artifactId>jmh-generator-annprocess</artifactId>
-	    <version>${jmh.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-core</artifactId>
+        <version>${jmh.version}</version>
         <scope>test</scope>
-	  </dependency>
+      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-generator-annprocess</artifactId>
+        <version>${jmh.version}</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -228,6 +229,34 @@
         </plugins>
       </build>
     </profile>
+
+    <!--
+      Profile that enables format checks on compilation
+      Can auto-format using spotless:apply.
+    -->
+    <profile>
+      <id>check-format</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.diffplug.spotless</groupId>
+            <artifactId>spotless-maven-plugin</artifactId>
+            <version>${spotless.version}</version>
+
+            <executions>
+              <execution>
+                <!-- Runs in initialize phase to fail fast in case of formatting issues (should be before codegen).-->
+                <id>spotless-check</id>
+                <phase>initialize</phase>
+                <goals>
+                  <goal>check</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <!-- http://central.sonatype.org/pages/requirements.html#developer-information -->
@@ -299,6 +328,22 @@
           <updateMatchingVersions>true</updateMatchingVersions>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <version>${spotless.version}</version>
+        <configuration>
+
+          <ratchetFrom>origin/master</ratchetFrom>
+
+          <lineEndings/>
+          <java>
+            <googleJavaFormat/>
+
+            <removeUnusedImports/>
+          </java>
+        </configuration>
+      </plugin>
     </plugins>
     <pluginManagement>
       <plugins>

tensorflow-core/pom.xml

@@ -102,9 +102,10 @@
       </activation>
       <modules>
         <module>tensorflow-core-platform</module>
-        <module>tensorflow-core-platform-mkl</module>
-        <module>tensorflow-core-platform-mkl-gpu</module>
         <module>tensorflow-core-platform-gpu</module>
+        <!-- Disabling MKL builds until they are more stable and performant -->
+        <!--module>tensorflow-core-platform-mkl</module-->
+        <!--module>tensorflow-core-platform-mkl-gpu</module-->
       </modules>
     </profile>
 

tensorflow-core/tensorflow-core-api/.bazelrc

@@ -151,8 +151,8 @@ build --define open_source_build=true
 test --define open_source_build=true
 
 # For workaround https://github.com/bazelbuild/bazel/issues/8772 with Bazel >= 0.29.1
-build --java_toolchain=@org_tensorflow//third_party/toolchains/java:tf_java_toolchain
-build --host_java_toolchain=@org_tensorflow//third_party/toolchains/java:tf_java_toolchain
+build --java_toolchain=@tf_toolchains//toolchains/java:tf_java_toolchain
+build --host_java_toolchain=@tf_toolchains//toolchains/java:tf_java_toolchain
 
 # Please note that MKL on MacOS or windows is still not supported.
 # If you would like to use a local MKL instead of downloading, please set the
@@ -616,3 +616,33 @@ build:release_gpu_linux_cuda_10_1 --config=release_gpu_linux
 build:release_gpu_linux_cuda_10_1 --action_env CUDA_TOOLKIT_PATH="/usr/local/cuda-10.1"
 build:release_gpu_linux_cuda_10_1 --action_env=TF_CUDA_VERSION="10"
 build:release_gpu_linux_cuda_10_1 --action_env=TF_CUDNN_VERSION="7"
+
+# Address sanitizer
+# CC=clang bazel build --config asan
+build:asan --strip=never
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --copt -g
+build:asan --copt -O3
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address
+
+# Memory sanitizer
+# CC=clang bazel build --config msan
+build:msan --strip=never
+build:msan --copt -fsanitize=memory
+build:msan --copt -DADDRESS_SANITIZER
+build:msan --copt -g
+build:msan --copt -O3
+build:msan --copt -fno-omit-frame-pointer
+build:msan --linkopt -fsanitize=memory
+
+# Undefined Behavior Sanitizer
+# CC=clang bazel build --config ubsan
+build:ubsan --strip=never
+build:ubsan --copt -fsanitize=undefined
+build:ubsan --copt -g
+build:ubsan --copt -O3
+build:ubsan --copt -fno-omit-frame-pointer
+build:ubsan --linkopt -fsanitize=undefined
+build:ubsan --linkopt -lubsan

tensorflow-core/tensorflow-core-api/WORKSPACE

@@ -17,28 +17,31 @@ http_archive(
     patch_args = ["-p1"],
     patch_cmds = ["grep -rl 'java_package' tensorflow/core | xargs sed -i.bak 's/^\(.* java_package = \"org\.tensorflow\.\)\(.*\"\)/\\1proto.\\2'/"],
     urls = [
-       "https://github.com/tensorflow/tensorflow/archive/v2.4.1.tar.gz",
+       "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.5.0.tar.gz",
     ],
-    sha256 = "f681331f8fc0800883761c7709d13cda11942d4ad5ff9f44ad855e9dc78387e0",
-    strip_prefix = "tensorflow-2.4.1"
+    sha256 = "233875ea27fc357f6b714b2a0de5f6ff124b50c1ee9b3b41f9e726e9e677b86c",
+    strip_prefix = "tensorflow-2.5.0"
 )
 
 # START: Upstream TensorFlow dependencies
 # TensorFlow build depends on these dependencies.
 # Needs to be in-sync with TensorFlow sources.
-http_archive(
-    name = "io_bazel_rules_closure",
-    sha256 = "5b00383d08dd71f28503736db0500b6fb4dda47489ff5fc6bed42557c07c6ba9",
-    strip_prefix = "rules_closure-308b05b2419edb5c8ee0471b67a40403df940149",
-    urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz",
-        "https://github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz",  # 2019-06-13
-    ],
-)
-# END: Upstream TensorFlow dependencies
+load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
+
+tf_workspace3()
 
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-tf_workspace()
+load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
+
+tf_workspace2()
+
+load("@org_tensorflow//tensorflow:workspace1.bzl", "tf_workspace1")
+
+tf_workspace1()
+
+load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0")
+
+tf_workspace0()
+# END: Upstream TensorFlow dependencies
 
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
 grpc_deps()

tensorflow-core/tensorflow-core-api/build.sh

@@ -68,12 +68,13 @@ done
 echo "Listing $TENSORFLOW_BIN:" && ls -l $TENSORFLOW_BIN
 
 if [[ -x /usr/bin/install_name_tool ]] && [[ -e $BAZEL_BIN/external/llvm_openmp/libiomp5.dylib ]]; then
-    # Fix library with correct rpath on Mac
-    chmod +w $BAZEL_BIN/external/llvm_openmp/libiomp5.dylib $TENSORFLOW_BIN/libtensorflow_cc.2.dylib $TENSORFLOW_BIN/libtensorflow_framework.2.dylib
-    UGLYPATH=$(otool -L $TENSORFLOW_BIN/libtensorflow_cc.2.dylib | grep @loader_path | cut -f1 -d ' ')
-    install_name_tool -add_rpath @loader_path/. -id @rpath/libiomp5.dylib $BAZEL_BIN/external/llvm_openmp/libiomp5.dylib
-    install_name_tool -change $UGLYPATH @rpath/libiomp5.dylib $TENSORFLOW_BIN/libtensorflow_cc.2.dylib
-    install_name_tool -change $UGLYPATH @rpath/libiomp5.dylib $TENSORFLOW_BIN/libtensorflow_framework.2.dylib
+   # Fix library with correct rpath on Mac
+   chmod +w $BAZEL_BIN/external/llvm_openmp/libiomp5.dylib $TENSORFLOW_BIN/libtensorflow_cc.2.dylib $TENSORFLOW_BIN/libtensorflow_framework.2.dylib
+   UGLYPATH=$(otool -L $TENSORFLOW_BIN/libtensorflow_cc.2.dylib | grep @loader_path | cut -f1 -d ' ')
+   echo $UGLYPATH
+   install_name_tool -add_rpath @loader_path/. -id @rpath/libiomp5.dylib $BAZEL_BIN/external/llvm_openmp/libiomp5.dylib
+   install_name_tool -change $UGLYPATH @rpath/libiomp5.dylib $TENSORFLOW_BIN/libtensorflow_cc.2.dylib
+   install_name_tool -change $UGLYPATH @rpath/libiomp5.dylib $TENSORFLOW_BIN/libtensorflow_framework.2.dylib
 fi
 
 GEN_SRCS_DIR=src/gen/java