From 18e1717a1354daa52f67f86d7c24e115891716ec Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 3 Oct 2022 21:23:35 +0200 Subject: [PATCH 1/2] Added datavec-data-excel Signed-off-by: brian --- cavis-common-platform/build.gradle | 5 +- .../build.gradle | 9 +++ .../datavec/poi/excel/ExcelRecordReader.java | 0 .../datavec/poi/excel/ExcelRecordWriter.java | 0 .../poi/excel/AssertTestsExtendBaseClass.java | 0 .../poi/excel/ExcelRecordReaderTest.java | 0 .../poi/excel/ExcelRecordWriterTest.java | 0 .../cavis-datavec-data/datavec-excel/pom.xml | 59 ------------------- cavis-dnn/cavis-dnn-api/build.gradle | 1 + settings.gradle | 1 + 10 files changed, 14 insertions(+), 61 deletions(-) rename cavis-datavec/cavis-datavec-data/{datavec-excel => cavis-datavec-data-excel}/build.gradle (74%) rename cavis-datavec/cavis-datavec-data/{datavec-excel => cavis-datavec-data-excel}/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java (100%) rename cavis-datavec/cavis-datavec-data/{datavec-excel => cavis-datavec-data-excel}/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java (100%) rename cavis-datavec/cavis-datavec-data/{datavec-excel => cavis-datavec-data-excel}/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java (100%) rename cavis-datavec/cavis-datavec-data/{datavec-excel => cavis-datavec-data-excel}/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java (100%) rename cavis-datavec/cavis-datavec-data/{datavec-excel => cavis-datavec-data-excel}/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java (100%) delete mode 100644 cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml diff --git a/cavis-common-platform/build.gradle b/cavis-common-platform/build.gradle index 7e0c7279f..05368a76b 100644 --- a/cavis-common-platform/build.gradle +++ b/cavis-common-platform/build.gradle @@ -9,7 +9,7 @@ ext { def javacpp = [version: "1.5.6", presetsVersion: "1.5.6"] def hdf5 = [version: "1.12.1"] - def jackson = [version: "2.10.5.20201202"] + def jackson = [version: "2.13.4"] def cuda = [version: "11.4"] def cudnn = [version: "8.2"] def openblas = [version: "0.3.17"] @@ -35,12 +35,13 @@ dependencies { api enforcedPlatform("io.netty:netty-bom:${netty.version}") api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jackson.version}") + //api enforcedPlatform("com.fasterxml.jackson.core:jackson-annotations:${jackson.version}") constraints { - api enforcedPlatform("io.netty:netty-bom:${netty.version}") api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jackson.version}") + //api enforcedPlatform("com.fasterxml.jackson.core:jackson-annotations:${jackson.version}") api 'com.google.guava:guava:30.1-jre' api "com.google.protobuf:protobuf-java:3.15.6" diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/build.gradle similarity index 74% rename from cavis-datavec/cavis-datavec-data/datavec-excel/build.gradle rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/build.gradle index 05af1969e..f7e4b2fc5 100644 --- a/cavis-datavec/cavis-datavec-data/datavec-excel/build.gradle +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/build.gradle @@ -18,9 +18,18 @@ * ***************************************************************************** * */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" dependencies { implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnCommon implementation "org.apache.poi:poi:3.17" implementation "org.apache.poi:poi-ooxml:3.17" + implementation "com.fasterxml.jackson.core:jackson-annotations" + + implementation "org.slf4j:slf4j-api" + + testRuntimeOnly 'net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT' + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests } \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java similarity index 100% rename from cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java similarity index 100% rename from cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java similarity index 100% rename from cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java similarity index 100% rename from cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java similarity index 100% rename from cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml b/cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml deleted file mode 100644 index dfde1d302..000000000 --- a/cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - 4.0.0 - - - net.brutex.ai - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-excel - - datavec-excel - - - - net.brutex.ai - datavec-api - ${project.version} - - - - org.apache.poi - poi - ${poi.version} - - - - org.apache.poi - poi-ooxml - ${poi.version} - - - - - diff --git a/cavis-dnn/cavis-dnn-api/build.gradle b/cavis-dnn/cavis-dnn-api/build.gradle index c0fc57e93..ffd000c1c 100644 --- a/cavis-dnn/cavis-dnn-api/build.gradle +++ b/cavis-dnn/cavis-dnn-api/build.gradle @@ -30,6 +30,7 @@ dependencies { implementation 'org.apache.commons:commons-lang3' implementation 'commons-io:commons-io' implementation 'org.slf4j:slf4j-api' + implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-annotations' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' diff --git a/settings.gradle b/settings.gradle index 4e8607f44..d6355b295 100644 --- a/settings.gradle +++ b/settings.gradle @@ -121,6 +121,7 @@ include ':cavis-datavec:cavis-datavec-data' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-image' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-audio' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-codec' +include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-excel' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-nlp' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-geo' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-hadoop' From 1749b9e4afa92f11f03889e73f671237e4dc771b Mon Sep 17 00:00:00 2001 From: brian Date: Tue, 4 Oct 2022 17:04:08 +0200 Subject: [PATCH 2/2] Fix compiler warnings Signed-off-by: brian --- .../main/java/org/deeplearning4j/eval/curves/RocCurve.java | 2 +- .../java/org/deeplearning4j/nn/conf/graph/LayerVertex.java | 1 - .../nn/conf/layers/misc/FrozenLayerWithBackprop.java | 2 ++ .../nn/conf/layers/samediff/SameDiffVertex.java | 2 ++ .../deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java | 6 ++---- .../nn/conf/layers/wrapper/BaseWrapperLayer.java | 2 ++ .../main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java | 2 +- .../java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java | 4 ++-- .../src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java | 3 ++- .../org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java | 6 +++--- .../java/org/nd4j/aeron/ipc/response/HostPortPublisher.java | 2 +- 11 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java index 17b176b9d..824115afa 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @Deprecated @Data -@EqualsAndHashCode(exclude = {"auc"}, callSuper = false) +@EqualsAndHashCode(callSuper = false) public class RocCurve extends org.nd4j.evaluation.curves.RocCurve { /** diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java index 677e3428c..b1734682d 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java @@ -36,7 +36,6 @@ import java.util.Arrays; @NoArgsConstructor @Data -@EqualsAndHashCode(callSuper = false) public class LayerVertex extends GraphVertex { private NeuralNetConfiguration layerConf; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java index f782673cb..53d7ff914 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.conf.layers.misc; import lombok.Data; +import lombok.EqualsAndHashCode; import org.deeplearning4j.nn.api.ParamInitializer; import org.deeplearning4j.nn.api.layers.LayerConstraint; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; @@ -38,6 +39,7 @@ import java.util.Collection; import java.util.List; @Data +@EqualsAndHashCode(callSuper = false) public class FrozenLayerWithBackprop extends BaseWrapperLayer { public FrozenLayerWithBackprop(@JsonProperty("layer") Layer layer) { diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java index b18aa1cc3..94a13ffec 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.conf.layers.samediff; import lombok.Data; +import lombok.EqualsAndHashCode; import org.deeplearning4j.nn.api.MaskState; import org.deeplearning4j.nn.api.TrainingConfig; import org.deeplearning4j.nn.conf.GradientNormalization; @@ -44,6 +45,7 @@ import java.util.List; import java.util.Map; @Data +@EqualsAndHashCode(callSuper = false) public abstract class SameDiffVertex extends GraphVertex implements TrainingConfig { private SDVertexParams vertexParams; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java index 410ea08c5..8a3d309a5 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java @@ -20,10 +20,7 @@ package org.deeplearning4j.nn.conf.layers.util; -import lombok.Data; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.*; import org.deeplearning4j.nn.conf.InputPreProcessor; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.inputs.InputType; @@ -38,6 +35,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collection; @Data +@EqualsAndHashCode(callSuper = false) public class MaskZeroLayer extends BaseWrapperLayer { private double maskingValue = 0.0; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java index 051a68968..ca90ee7a1 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.conf.layers.wrapper; import lombok.Data; +import lombok.EqualsAndHashCode; import org.deeplearning4j.nn.api.ParamInitializer; import org.deeplearning4j.nn.conf.GradientNormalization; import org.deeplearning4j.nn.conf.InputPreProcessor; @@ -33,6 +34,7 @@ import org.nd4j.linalg.learning.regularization.Regularization; import java.util.List; @Data +@EqualsAndHashCode(callSuper = false) public abstract class BaseWrapperLayer extends Layer { protected Layer underlying; diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java index 20affc794..9e8d1660f 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java @@ -61,7 +61,7 @@ public class AeronNDArrayPublisher implements AutoCloseable { public final static int NUM_RETRIES = 100; private boolean compress = true; private static final BusySpinIdleStrategy busySpinIdleStrategy = new BusySpinIdleStrategy(); - private int publishRetryTimeOut = 3000; + @Builder.Default private int publishRetryTimeOut = 3000; private void init() { channel = channel == null ? "aeron:udp?endpoint=localhost:40123" : channel; diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java index 91a3dbea5..204893e97 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java @@ -45,7 +45,7 @@ public class AeronNDArraySubscriber implements AutoCloseable { private String channel; // A unique identifier for a stream within a channel. Stream ID 0 is reserved // for internal use and should not be used by applications. - private int streamId = -1; + @Builder.Default private int streamId = -1; // Maximum number of message fragments to receive during a single 'poll' operation private int fragmentLimitCount; // Create a context, needed for client connection to media driver @@ -57,7 +57,7 @@ public class AeronNDArraySubscriber implements AutoCloseable { private NDArrayCallback ndArrayCallback; private Aeron aeron; private Subscription subscription; - private AtomicBoolean launched = new AtomicBoolean(false); + @Builder.Default private AtomicBoolean launched = new AtomicBoolean(false); private Executor executors; diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java index 13022f78a..3658a8006 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java @@ -50,7 +50,8 @@ public class NDArrayMessage implements Serializable { private long index; private int[] dimensions; private byte[] chunk; - private int numChunks = 0; + + @Builder.Default private int numChunks = 0; //default dimensions: a 1 length array of -1 means use the whole array for an update. private static int[] WHOLE_ARRAY_UPDATE = {-1}; //represents the constant for indicating using the whole array for an update (-1) diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java index d71ba3992..cea7193e8 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java @@ -43,17 +43,17 @@ public class AeronNDArrayResponder implements AutoCloseable { private String channel; // A unique identifier for a stream within a channel. Stream ID 0 is reserved // for internal use and should not be used by applications. - private int streamId = -1; + @Builder.Default private int streamId = -1; // A unique identifier for a stream within a channel. Stream ID 0 is reserved // for internal use and should not be used by applications. - private int responseStreamId = -1; + @Builder.Default private int responseStreamId = -1; // Maximum number of message fragments to receive during a single 'poll' operation private int fragmentLimitCount; // Create a context, needed for client connection to media driver // A separate media driver process need to run prior to running this application private Aeron.Context ctx; - private AtomicBoolean running = new AtomicBoolean(true); + @Builder.Default private AtomicBoolean running = new AtomicBoolean(true); private final AtomicBoolean init = new AtomicBoolean(false); private static Logger log = LoggerFactory.getLogger(AeronNDArrayResponder.class); private NDArrayHolder ndArrayHolder; diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java index 7a9c5826c..083f78a7d 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java @@ -42,7 +42,7 @@ public class HostPortPublisher implements AutoCloseable { private int streamId; // The channel (an endpoint identifier) to send the message to private String channel; - private boolean init = false; + @Builder.Default private boolean init = false; private Aeron.Context ctx; private Aeron aeron; private Publication publication;