Merge branch 'master' into enhance-build-infrastructure

cavis-common-platform/build.gradle

@@ -9,7 +9,7 @@ ext {
 
     def javacpp = [version: "1.5.6", presetsVersion: "1.5.6"]
     def hdf5 = [version: "1.12.1"]
-    def jackson = [version: "2.10.5.20201202"]
+    def jackson = [version: "2.13.4"]
     def cuda = [version: "11.4"]
     def cudnn = [version: "8.2"]
     def openblas = [version: "0.3.17"]
@@ -35,12 +35,13 @@ dependencies {
 
     api enforcedPlatform("io.netty:netty-bom:${netty.version}")
     api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jackson.version}")
+    //api enforcedPlatform("com.fasterxml.jackson.core:jackson-annotations:${jackson.version}")
 
 
     constraints {
-
         api enforcedPlatform("io.netty:netty-bom:${netty.version}")
         api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jackson.version}")
+        //api enforcedPlatform("com.fasterxml.jackson.core:jackson-annotations:${jackson.version}")
 
         api 'com.google.guava:guava:30.1-jre'
         api "com.google.protobuf:protobuf-java:3.15.6"

cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/build.gradle

@@ -18,9 +18,18 @@
  *    *****************************************************************************
  *
  */
+apply from: "${project.rootProject.projectDir}/createTestBackends.gradle"
 
 dependencies {
     implementation projects.cavisDatavec.cavisDatavecApi
+    implementation projects.cavisDnn.cavisDnnCommon
     implementation "org.apache.poi:poi:3.17"
     implementation "org.apache.poi:poi-ooxml:3.17"
+    implementation "com.fasterxml.jackson.core:jackson-annotations"
+
+    implementation "org.slf4j:slf4j-api"
+
+    testRuntimeOnly 'net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT'
+
+    testImplementation projects.cavisNd4j.cavisNd4jCommonTests
 }

cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml

@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ /* ******************************************************************************
-  ~  *
-  ~  *
-  ~  * This program and the accompanying materials are made available under the
-  ~  * terms of the Apache License, Version 2.0 which is available at
-  ~  * https://www.apache.org/licenses/LICENSE-2.0.
-  ~  *
-  ~  *  See the NOTICE file distributed with this work for additional
-  ~  *  information regarding copyright ownership.
-  ~  * Unless required by applicable law or agreed to in writing, software
-  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-  ~  * License for the specific language governing permissions and limitations
-  ~  * under the License.
-  ~  *
-  ~  * SPDX-License-Identifier: Apache-2.0
-  ~  ******************************************************************************/
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>net.brutex.ai</groupId>
-        <artifactId>datavec-parent</artifactId>
-        <version>1.0.0-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>datavec-excel</artifactId>
-
-    <name>datavec-excel</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>datavec-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi -->
-        <dependency>
-            <groupId>org.apache.poi</groupId>
-            <artifactId>poi</artifactId>
-            <version>${poi.version}</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
-        <dependency>
-            <groupId>org.apache.poi</groupId>
-            <artifactId>poi-ooxml</artifactId>
-            <version>${poi.version}</version>
-        </dependency>
-    </dependencies>
-
-
-</project>

cavis-dnn/cavis-dnn-api/build.gradle

@@ -30,6 +30,7 @@ dependencies {
     implementation 'org.apache.commons:commons-lang3'
     implementation 'commons-io:commons-io'
     implementation 'org.slf4j:slf4j-api'
+    implementation 'com.fasterxml.jackson.core:jackson-core'
     implementation 'com.fasterxml.jackson.core:jackson-annotations'
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'

cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java

@@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 
 @Deprecated
 @Data
-@EqualsAndHashCode(exclude = {"auc"}, callSuper = false)
+@EqualsAndHashCode(callSuper = false)
 public class RocCurve extends org.nd4j.evaluation.curves.RocCurve {
 
     /**

cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java

@@ -36,7 +36,6 @@ import java.util.Arrays;
 
 @NoArgsConstructor
 @Data
-@EqualsAndHashCode(callSuper = false)
 public class LayerVertex extends GraphVertex {
 
     private NeuralNetConfiguration layerConf;

cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java

@@ -21,6 +21,7 @@
 package org.deeplearning4j.nn.conf.layers.misc;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import org.deeplearning4j.nn.api.ParamInitializer;
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
 import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
@@ -38,6 +39,7 @@ import java.util.Collection;
 import java.util.List;
 
 @Data
+@EqualsAndHashCode(callSuper = false)
 public class FrozenLayerWithBackprop extends BaseWrapperLayer {
 
     public FrozenLayerWithBackprop(@JsonProperty("layer") Layer layer) {

cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java

@@ -21,6 +21,7 @@
 package org.deeplearning4j.nn.conf.layers.samediff;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import org.deeplearning4j.nn.api.MaskState;
 import org.deeplearning4j.nn.api.TrainingConfig;
 import org.deeplearning4j.nn.conf.GradientNormalization;
@@ -44,6 +45,7 @@ import java.util.List;
 import java.util.Map;
 
 @Data
+@EqualsAndHashCode(callSuper = false)
 public abstract class SameDiffVertex extends GraphVertex implements TrainingConfig {
 
     private SDVertexParams vertexParams;

cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java

@@ -20,10 +20,7 @@
 
 package org.deeplearning4j.nn.conf.layers.util;
 
-import lombok.Data;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+import lombok.*;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
 import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
 import org.deeplearning4j.nn.conf.inputs.InputType;
@@ -38,6 +35,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Collection;
 
 @Data
+@EqualsAndHashCode(callSuper = false)
 public class MaskZeroLayer extends BaseWrapperLayer {
 
     private double maskingValue = 0.0;

cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java

@@ -21,6 +21,7 @@
 package org.deeplearning4j.nn.conf.layers.wrapper;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import org.deeplearning4j.nn.api.ParamInitializer;
 import org.deeplearning4j.nn.conf.GradientNormalization;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
@@ -33,6 +34,7 @@ import org.nd4j.linalg.learning.regularization.Regularization;
 import java.util.List;
 
 @Data
+@EqualsAndHashCode(callSuper = false)
 public abstract class BaseWrapperLayer extends Layer {
 
     protected Layer underlying;

cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java

@@ -61,7 +61,7 @@ public class AeronNDArrayPublisher implements AutoCloseable {
     public final static int NUM_RETRIES = 100;
     private boolean compress = true;
     private static final BusySpinIdleStrategy busySpinIdleStrategy = new BusySpinIdleStrategy();
-    private int publishRetryTimeOut = 3000;
+    @Builder.Default private int publishRetryTimeOut = 3000;
 
     private void init() {
         channel = channel == null ? "aeron:udp?endpoint=localhost:40123" : channel;

cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java

@@ -45,7 +45,7 @@ public class AeronNDArraySubscriber implements AutoCloseable {
     private String channel;
     // A unique identifier for a stream within a channel. Stream ID 0 is reserved
     // for internal use and should not be used by applications.
-    private int streamId = -1;
+    @Builder.Default private int streamId = -1;
     // Maximum number of message fragments to receive during a single 'poll' operation
     private int fragmentLimitCount;
     // Create a context, needed for client connection to media driver
@@ -57,7 +57,7 @@ public class AeronNDArraySubscriber implements AutoCloseable {
     private NDArrayCallback ndArrayCallback;
     private Aeron aeron;
     private Subscription subscription;
-    private AtomicBoolean launched = new AtomicBoolean(false);
+    @Builder.Default private AtomicBoolean launched = new AtomicBoolean(false);
     private Executor executors;
 
 

cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java

@@ -50,7 +50,8 @@ public class NDArrayMessage implements Serializable {
     private long index;
     private int[] dimensions;
     private byte[] chunk;
-    private int numChunks = 0;
+
+    @Builder.Default private int numChunks = 0;
     //default dimensions: a 1 length array of -1 means use the whole array for an update.
     private static int[] WHOLE_ARRAY_UPDATE = {-1};
     //represents the constant for indicating using the whole array for an update (-1)

cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java

@@ -43,17 +43,17 @@ public class AeronNDArrayResponder implements AutoCloseable {
     private String channel;
     // A unique identifier for a stream within a channel. Stream ID 0 is reserved
     // for internal use and should not be used by applications.
-    private int streamId = -1;
+    @Builder.Default private int streamId = -1;
     // A unique identifier for a stream within a channel. Stream ID 0 is reserved
     // for internal use and should not be used by applications.
-    private int responseStreamId = -1;
+    @Builder.Default private int responseStreamId = -1;
 
     // Maximum number of message fragments to receive during a single 'poll' operation
     private int fragmentLimitCount;
     // Create a context, needed for client connection to media driver
     // A separate media driver process need to run prior to running this application
     private Aeron.Context ctx;
-    private AtomicBoolean running = new AtomicBoolean(true);
+    @Builder.Default private AtomicBoolean running = new AtomicBoolean(true);
     private final AtomicBoolean init = new AtomicBoolean(false);
     private static Logger log = LoggerFactory.getLogger(AeronNDArrayResponder.class);
     private NDArrayHolder ndArrayHolder;

cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java

@@ -42,7 +42,7 @@ public class HostPortPublisher implements AutoCloseable {
     private int streamId;
     // The channel (an endpoint identifier) to send the message to
     private String channel;
-    private boolean init = false;
+    @Builder.Default private boolean init = false;
     private Aeron.Context ctx;
     private Aeron aeron;
     private Publication publication;

settings.gradle

@@ -121,6 +121,7 @@ include ':cavis-datavec:cavis-datavec-data'
 include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-image'
 include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-audio'
 include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-codec'
+include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-excel'
 include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-nlp'
 include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-geo'
 include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-hadoop'