Merge pull request #9215 from eclipse/ag_build_fixes

Build fixes for javacpp versions, empty snapshot jars

.github/workflows/build-android-x86_64.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   android-x86_64:
     runs-on: ubuntu-18.04

.github/workflows/build-deploy-android-arm32.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
   android-arm32:

.github/workflows/build-deploy-android-arm64.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
   android-arm64:

.github/workflows/build-deploy-linux-arm32.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
   linux-arm32:

.github/workflows/build-deploy-linux-arm64.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
   linux-arm64:

.github/workflows/build-deploy-linux-cuda-11.0.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
 
   linux-x86_64-cuda_11-0:

.github/workflows/build-deploy-linux-cuda-11.2.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   linux-x86_64-cuda-11-2:
     runs-on: ubuntu-18.04

.github/workflows/build-deploy-linux-x86_64.yml

@@ -1,6 +1,8 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
+  push:
 jobs:
   #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
   linux-x86_64:

.github/workflows/build-deploy-mac.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   mac-x86_64:
     runs-on: macos-10.15

.github/workflows/build-deploy-windows-cuda-11.0.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   windows-x86_64-cuda-11-0:
     runs-on: windows-2019

.github/workflows/build-deploy-windows-cuda-11.2.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
   windows-x86_64-cuda-11-2:
     runs-on: windows-2019

.github/workflows/build-deploy-windows.yml

@@ -1,6 +1,7 @@
 on:
   schedule:
     - cron: "0 */12 * * *"
+  workflow_dispatch:
 jobs:
 
   windows-x86_64:

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaLayer.java

@@ -30,11 +30,11 @@ import java.util.Map;
 public abstract class SameDiffLambdaLayer extends SameDiffLayer {
 
     /**
-     * The defineLayer method is used to define the foward pass for the layer
+     * The defineLayer method is used to define the forward pass for the layer
      *
      * @param sameDiff   SameDiff instance to use to define the vertex
      * @param layerInput Layer input variable
-     * @return The output variable (orresponding to the output activations for the layer)
+     * @return The output variable (corresponding to the output activations for the layer)
      */
     public abstract SDVariable defineLayer(SameDiff sameDiff, SDVariable layerInput);
 

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/ir/TensorNamespace.java

@@ -5411,12 +5411,14 @@ public final class TensorNamespace {
      * Serializations can either use one of the fields above, or use this
      * raw bytes field. The only exception is the string case, where one is
      * required to store the content in the repeated bytes string_data field.
+     *
      * When this raw_data field is used to store tensor value, elements MUST
      * be stored in as fixed-width, little-endian order.
      * Floating-point data types MUST be stored in IEEE 754 format.
      * Complex64 elements must be written as two consecutive FLOAT values, real component first.
      * Complex128 elements must be written as two consecutive DOUBLE values, real component first.
      * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false).
+     *
      * Note: the advantage of specific field rather than the raw_data field is
      * that in some cases (e.g. int data), protobuf does a better packing via
      * variable length storage, and may lead to smaller binary footprint.
@@ -5655,6 +5657,7 @@ public final class TensorNamespace {
   /**
    * <pre>
    * Tensors
+   *
    * A serialized tensor value.
    * </pre>
    *
@@ -7010,12 +7013,14 @@ public final class TensorNamespace {
      * Serializations can either use one of the fields above, or use this
      * raw bytes field. The only exception is the string case, where one is
      * required to store the content in the repeated bytes string_data field.
+     *
      * When this raw_data field is used to store tensor value, elements MUST
      * be stored in as fixed-width, little-endian order.
      * Floating-point data types MUST be stored in IEEE 754 format.
      * Complex64 elements must be written as two consecutive FLOAT values, real component first.
      * Complex128 elements must be written as two consecutive DOUBLE values, real component first.
      * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false).
+     *
      * Note: the advantage of specific field rather than the raw_data field is
      * that in some cases (e.g. int data), protobuf does a better packing via
      * variable length storage, and may lead to smaller binary footprint.
@@ -7766,6 +7771,7 @@ public final class TensorNamespace {
     /**
      * <pre>
      * Tensors
+     *
      * A serialized tensor value.
      * </pre>
      *
@@ -9080,12 +9086,14 @@ public final class TensorNamespace {
        * Serializations can either use one of the fields above, or use this
        * raw bytes field. The only exception is the string case, where one is
        * required to store the content in the repeated bytes string_data field.
+       *
        * When this raw_data field is used to store tensor value, elements MUST
        * be stored in as fixed-width, little-endian order.
        * Floating-point data types MUST be stored in IEEE 754 format.
        * Complex64 elements must be written as two consecutive FLOAT values, real component first.
        * Complex128 elements must be written as two consecutive DOUBLE values, real component first.
        * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false).
+       *
        * Note: the advantage of specific field rather than the raw_data field is
        * that in some cases (e.g. int data), protobuf does a better packing via
        * variable length storage, and may lead to smaller binary footprint.
@@ -9102,12 +9110,14 @@ public final class TensorNamespace {
        * Serializations can either use one of the fields above, or use this
        * raw bytes field. The only exception is the string case, where one is
        * required to store the content in the repeated bytes string_data field.
+       *
        * When this raw_data field is used to store tensor value, elements MUST
        * be stored in as fixed-width, little-endian order.
        * Floating-point data types MUST be stored in IEEE 754 format.
        * Complex64 elements must be written as two consecutive FLOAT values, real component first.
        * Complex128 elements must be written as two consecutive DOUBLE values, real component first.
        * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false).
+       *
        * Note: the advantage of specific field rather than the raw_data field is
        * that in some cases (e.g. int data), protobuf does a better packing via
        * variable length storage, and may lead to smaller binary footprint.
@@ -9130,12 +9140,14 @@ public final class TensorNamespace {
        * Serializations can either use one of the fields above, or use this
        * raw bytes field. The only exception is the string case, where one is
        * required to store the content in the repeated bytes string_data field.
+       *
        * When this raw_data field is used to store tensor value, elements MUST
        * be stored in as fixed-width, little-endian order.
        * Floating-point data types MUST be stored in IEEE 754 format.
        * Complex64 elements must be written as two consecutive FLOAT values, real component first.
        * Complex128 elements must be written as two consecutive DOUBLE values, real component first.
        * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false).
+       *
        * Note: the advantage of specific field rather than the raw_data field is
        * that in some cases (e.g. int data), protobuf does a better packing via
        * variable length storage, and may lead to smaller binary footprint.

nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml

@@ -255,7 +255,7 @@
                             <classOrPackageName>org.nd4j.nativeblas.Nd4jCuda</classOrPackageName>
                             <copyLibs>true</copyLibs>
                             <configDirectory>${project.build.directory}/classes/META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/</configDirectory>
-                            <outputDirectory>${project.build.directory}</outputDirectory>
+                            <outputDirectory>${project.build.directory}/classes/org/nd4j/nativeblas/${javacpp.platform}</outputDirectory>
                         </configuration>
                     </execution>
                 </executions>
@@ -325,6 +325,42 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>default-deploy</id>
+                        <phase>none</phase>
+                    </execution>
+
+                    <execution>
+                        <id>java-classes</id>
+                        <phase>deploy</phase>
+                        <configuration>
+                            <packaging>jar</packaging>
+                            <artifactId>${project.artifactId}</artifactId>
+                            <groupId>${project.groupId}</groupId>
+                            <version>${project.version}</version>
+                            <sources>${project.build.directory}/nd4j-cuda-${cuda.version}-${project.version}.jar</sources>
+                        </configuration>
+                    </execution>
+
+                    <execution>
+                        <id>native-deps</id>
+                        <phase>deploy</phase>
+                        <configuration>
+                            <packaging>jar</packaging>
+                            <artifactId>${project.artifactId}</artifactId>
+                            <groupId>${project.groupId}</groupId>
+                            <version>${project.version}</version>
+                            <classifier>${javacpp.platform}</classifier>
+                            <sources>${project.build.directory}/nd4j-cuda-${cuda.version}-${project.version}-${javacpp.platform}.jar</sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
         </plugins>
     </build>
 

nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/pom.xml

@@ -93,7 +93,7 @@
                         <configuration>
                             <skipMain>${javacpp.parser.skip}</skipMain>
                             <includes>
-                                <include>org/nd4j/nativeblas/**.java</include>
+                                <include>org/nd4j/nativeblas/*.java</include>
                             </includes>
                         </configuration>
                     </execution>
@@ -219,14 +219,12 @@
                             <classOrPackageName>org.nd4j.nativeblas.Nd4jCpu</classOrPackageName>
                             <copyLibs>true</copyLibs>
                             <configDirectory>${project.build.directory}/classes/META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/</configDirectory>
-                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
+                            <outputDirectory>${project.build.directory}/classes/org/nd4j/nativeblas/${javacpp.platform}</outputDirectory>
                         </configuration>
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
+
-                <artifactId>maven-jar-plugin</artifactId>
-            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-enforcer-plugin</artifactId>
@@ -261,6 +259,43 @@
                     </execution>
                 </executions>
             </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>default-deploy</id>
+                        <phase>none</phase>
+                    </execution>
+
+                    <execution>
+                        <id>java-classes</id>
+                        <phase>deploy</phase>
+                        <configuration>
+                            <packaging>jar</packaging>
+                            <artifactId>${project.artifactId}</artifactId>
+                            <groupId>${project.groupId}</groupId>
+                            <version>${project.version}</version>
+                            <sources>${project.build.directory}/nd4j-native-${project.version}.jar</sources>
+                        </configuration>
+                    </execution>
+
+                    <execution>
+                        <id>native-deps</id>
+                        <phase>deploy</phase>
+                        <configuration>
+                            <packaging>jar</packaging>
+                            <artifactId>${project.artifactId}</artifactId>
+                            <groupId>${project.groupId}</groupId>
+                            <version>${project.version}</version>
+                            <classifier>${javacpp.platform}</classifier>
+                            <sources>${project.build.directory}/nd4j-native-${project.version}-${javacpp.platform}.jar</sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
         </plugins>
     </build>
 

nd4j/nd4j-backends/nd4j-backend-impls/pom.xml

@@ -166,6 +166,9 @@
                                         org/nd4j/nativeblas/${javacpp.platform}${javacpp.platform.extension}/*
                                     </exclude>
                                     <exclude>lib/**</exclude>
+                                    <exclude>*.dll</exclude>
+                                    <exclude>*.dylib</exclude>
+                                    <exclude>*.so</exclude>
                                     <exclude>META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/</exclude>
                                 </excludes>
                             </configuration>

nd4j/nd4j-onnxruntime/pom.xml

@@ -38,7 +38,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
-        <onnxruntime.version>1.6.0</onnxruntime.version>
+        <onnxruntime.version>1.7.0</onnxruntime.version>
         <onnxruntime.javacpp.version>${onnxruntime.version}-${javacpp.version}</onnxruntime.javacpp.version>
     </properties>
 

nd4j/samediff-import/pom.xml

@@ -22,6 +22,12 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.nd4j</groupId>
+    <artifactId>nd4j</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+  </parent>
+
 
   <groupId>org.nd4j</groupId>
   <artifactId>samediff-import</artifactId>
@@ -63,8 +69,6 @@
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.12</version>
-      <scope>test</scope>
     </dependency>
     <!-- Test Dependencies -->
     <dependency>
@@ -83,13 +87,10 @@
     <dependency>
       <groupId>org.jetbrains.kotlin</groupId>
       <artifactId>kotlin-stdlib-jdk8</artifactId>
-      <version>${kotlin.version}</version>
     </dependency>
     <dependency>
       <groupId>org.jetbrains.kotlin</groupId>
       <artifactId>kotlin-test</artifactId>
-      <version>${kotlin.version}</version>
-      <scope>test</scope>
     </dependency>
   </dependencies>
 
@@ -152,54 +153,6 @@
         <plugin>
           <groupId>org.jetbrains.kotlin</groupId>
           <artifactId>kotlin-maven-plugin</artifactId>
-          <version>${kotlin.version}</version>
-          <configuration>
-            <args>
-              <arg>-Xjsr305=strict</arg>
-            </args>
-            <compilerPlugins>
-              <plugin>spring</plugin>
-              <plugin>jpa</plugin>
-            </compilerPlugins>
-          </configuration>
-          <dependencies>
-            <dependency>
-              <groupId>org.jetbrains.kotlin</groupId>
-              <artifactId>kotlin-maven-allopen</artifactId>
-              <version>1.4.30-M1</version>
-            </dependency>
-            <dependency>
-              <groupId>org.jetbrains.kotlin</groupId>
-              <artifactId>kotlin-maven-noarg</artifactId>
-              <version>1.4.30-M1</version>
-            </dependency>
-          </dependencies>
-          <executions>
-            <execution>
-              <id>compile</id>
-              <goals> <goal>compile</goal> </goals>
-              <configuration>
-                <sourceDirs>
-                  <sourceDir>${project.basedir}/src/main/stubs</sourceDir>
-                  <sourceDir>${project.basedir}/src/main/kotlin</sourceDir>
-                  <sourceDir>${project.basedir}/src/main/java</sourceDir>
-                  <sourceDir>${project.basedir}/src/main/ops</sourceDir>
-                </sourceDirs>
-              </configuration>
-            </execution>
-            <execution>
-              <id>test-compile</id>
-              <goals> <goal>test-compile</goal> </goals>
-              <configuration>
-                <sourceDirs>
-                  <sourceDir>${project.basedir}/src/test/stubs</sourceDir>
-                  <sourceDir>${project.basedir}/src/test/kotlin</sourceDir>
-                  <sourceDir>${project.basedir}/src/test/java</sourceDir>
-                  <sourceDir>${project.basedir}/src/test/ops</sourceDir>
-                </sourceDirs>
-              </configuration>
-            </execution>
-          </executions>
         </plugin>
 
         <!-- https://kotlinlang.org/docs/reference/using-maven.html -->

nd4j/samediff-import/samediff-import-api/pom.xml

@@ -94,11 +94,7 @@
       <version>${commonsio.version}</version>
     </dependency>
 
-    <dependency>
+
-      <groupId>org.projectlombok</groupId>
-      <artifactId>lombok</artifactId>
-      <version>${lombok.version}</version>
-    </dependency>
 
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -132,7 +128,6 @@
     <dependency>
       <groupId>org.nd4j</groupId>
       <artifactId>nd4j-api</artifactId>
-      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.nd4j</groupId>
@@ -144,6 +139,7 @@
       <groupId>org.nd4j</groupId>
       <artifactId>nd4j-native</artifactId>
       <version>${project.version}</version>
+       <scope>test</scope>
     </dependency>
 
     <dependency>

nd4j/samediff-import/samediff-import-api/src/main/kotlin/org/nd4j/samediff/frameworkimport/IRProtobufExtensions.kt

@@ -33,7 +33,6 @@ import org.nd4j.ir.TensorNamespace
 import org.nd4j.linalg.api.buffer.DataType
 import org.nd4j.linalg.api.ndarray.INDArray
 import org.nd4j.linalg.factory.Nd4j
-import org.nd4j.nativeblas.Nd4jCpu.FLOAT32
 import org.nd4j.samediff.frameworkimport.opdefs.OpDescriptorLoaderHolder
 import org.nd4j.shade.protobuf.ByteString
 import java.lang.IllegalArgumentException

pom.xml

@@ -186,7 +186,7 @@
         <nd4j.cuda.basedir />
 
 
-        <python.version>3.9.1</python.version>
+        <python.version>3.9.2</python.version>
         <cpython-platform.version>${python.version}-${javacpp-presets.version}</cpython-platform.version>
         <numpy.version>1.20.1</numpy.version>
         <numpy.javacpp.version>${numpy.version}-${javacpp-presets.version}</numpy.javacpp.version>
@@ -194,7 +194,7 @@
         <openblas.version>0.3.13</openblas.version>
         <mkl.version>2021.1</mkl.version>
         <opencv.version>4.5.1</opencv.version>
-        <ffmpeg.version>4.3.1</ffmpeg.version>
+        <ffmpeg.version>4.3.2</ffmpeg.version>
         <leptonica.version>1.80.0</leptonica.version>
         <hdf5.version>1.12.0</hdf5.version>
         <ale.version>0.6.1</ale.version>