From 11ba7a59c14bf0cd074ab3bdb04025f73acf93b8 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 6 Oct 2022 13:22:06 +0200 Subject: [PATCH] More test fixes --- brutex-extended-tests/build.gradle | 2 + .../integration/IntegrationTestRunner.java | 4 + .../integration/IntegrationTestsDL4J.java | 8 +- .../integration/IntegrationTestsSameDiff.java | 6 -- cavis-common-platform/build.gradle | 1 + .../NDArrayColumnsMathOpTransform.java | 2 + .../ndarray/NDArrayMathFunctionTransform.java | 2 + .../ndarray/NDArrayScalarOpTransform.java | 2 + .../StringListToIndicesNDArrayTransform.java | 2 + .../cavis-datavec-data-geo/build.gradle | 1 + .../org/datavec/image/loader/LFWLoader.java | 2 +- .../image/loader/NativeImageLoader.java | 83 +++++++++++++------ .../recordreader/BaseImageRecordReader.java | 7 +- .../image/transform/BoxImageTransform.java | 2 + .../image/transform/FlipImageTransform.java | 2 + .../transform/LargestBlobCropTransform.java | 2 + .../image/transform/MultiImageTransform.java | 2 + .../transform/PipelineImageTransform.java | 2 + .../image/transform/RandomCropTransform.java | 2 + .../image/loader/TestNativeImageLoader.java | 36 -------- .../functions/DifferentialFunction.java | 5 +- .../nd4j/autodiff/samediff/SDVariable.java | 14 +--- .../autodiff/samediff/internal/Variable.java | 1 + .../org/nd4j/autodiff/util/SameDiffUtils.java | 2 +- .../classification/EvaluationCalibration.java | 2 +- .../org/nd4j/evaluation/curves/Histogram.java | 2 + .../deallocation/DeallocatableReference.java | 3 + .../linalg/api/ops/BaseIndexAccumulation.java | 2 + .../nd4j/linalg/api/ops/DynamicCustomOp.java | 3 + .../nd4j/linalg/api/ops/custom/Flatten.java | 2 + .../impl/controlflow/compat/BaseCompatOp.java | 3 + .../ops/impl/controlflow/compat/Enter.java | 2 + .../ops/impl/controlflow/compat/While.java | 2 + .../api/ops/impl/indexaccum/FirstIndex.java | 2 + .../api/ops/impl/indexaccum/LastIndex.java | 2 + .../ops/impl/indexaccum/custom/ArgAmax.java | 2 + .../ops/impl/indexaccum/custom/ArgAmin.java | 2 + .../ops/impl/indexaccum/custom/ArgMax.java | 2 + .../ops/impl/indexaccum/custom/ArgMin.java | 2 + .../convolution/config/Conv1DConfig.java | 7 +- .../convolution/config/Conv2DConfig.java | 2 + .../convolution/config/Conv3DConfig.java | 2 + .../convolution/config/DeConv2DConfig.java | 2 + .../convolution/config/DeConv3DConfig.java | 2 + .../LocalResponseNormalizationConfig.java | 2 + .../convolution/config/Pooling2DConfig.java | 2 + .../convolution/config/Pooling3DConfig.java | 2 + .../nd4j/linalg/api/ops/impl/reduce/Mmul.java | 2 +- .../linalg/api/ops/impl/reduce/MmulBp.java | 2 +- .../api/ops/impl/reduce/custom/BatchMmul.java | 2 +- .../linalg/dataset/BalanceMinibatches.java | 5 ++ .../nd4j/linalg/learning/config/AdaMax.java | 6 +- .../org/nd4j/linalg/profiler/OpProfiler.java | 14 ---- .../java/org/deeplearning4j/BaseDL4JTest.java | 7 -- cavis-dnn/cavis-dnn-common/build.gradle | 4 +- .../deeplearning4j/datasets/TestDataSets.java | 5 -- .../fetchers/SvhnDataFetcherTest.java | 5 -- .../iterator/DataSetIteratorTest.java | 5 -- .../earlystopping/TestEarlyStopping.java | 2 + .../gradientcheck/AttentionLayerTest.java | 4 - .../gradientcheck/BNGradientCheckTest.java | 5 -- .../gradientcheck/CNN1DGradientCheckTest.java | 5 -- .../gradientcheck/CNN3DGradientCheckTest.java | 5 -- .../gradientcheck/CNNGradientCheckTest.java | 5 -- .../CapsnetGradientCheckTest.java | 5 -- .../gradientcheck/DropoutGradientCheck.java | 5 -- .../GlobalPoolingGradientCheckTests.java | 5 -- .../gradientcheck/GradientCheckTests.java | 5 -- .../GradientCheckTestsComputationGraph.java | 5 -- .../GradientCheckTestsMasking.java | 5 -- .../gradientcheck/LRNGradientCheckTests.java | 6 -- .../gradientcheck/LSTMGradientCheckTests.java | 5 -- .../LossFunctionGradientCheck.java | 5 -- .../NoBiasGradientCheckTests.java | 5 -- .../OutputLayerGradientChecks.java | 5 -- .../gradientcheck/RnnGradientChecks.java | 5 -- .../UtilLayerGradientChecks.java | 5 -- .../gradientcheck/VaeGradientCheckTests.java | 5 -- .../gradientcheck/YoloGradientCheckTests.java | 5 -- .../deeplearning4j/nn/dtypes/DTypeTests.java | 5 -- .../custom/testclasses/CustomActivation.java | 2 +- .../normalization/BatchNormalizationTest.java | 5 -- .../testlayers/SameDiffDenseVertex.java | 2 + .../nn/multilayer/MultiLayerTest.java | 2 + .../EncodedGradientsAccumulatorTest.java | 5 -- .../SmartFancyBlockingQueueTest.java | 2 +- .../listener/TestCheckpointListener.java | 5 -- .../optimizer/listener/TestListeners.java | 5 -- .../regressiontest/RegressionTest060.java | 5 -- .../regressiontest/RegressionTest071.java | 5 -- .../regressiontest/RegressionTest080.java | 5 -- .../regressiontest/RegressionTest100a.java | 5 -- .../regressiontest/RegressionTest100b3.java | 5 -- .../regressiontest/RegressionTest100b4.java | 5 -- .../regressiontest/RegressionTest100b6.java | 5 -- .../TestDistributionDeserializer.java | 5 -- .../build.gradle | 1 + .../keras/layers/TFOpLayerImpl.java | 2 + .../keras/layers/core/KerasMasking.java | 2 + .../keras/layers/core/KerasMerge.java | 2 + .../clustering/vptree/VPTree.java | 4 +- .../deeplearning4j/eval/curves/Histogram.java | 2 + .../ParameterAveragingTrainingMaster.java | 3 +- .../training/SharedTrainingMaster.java | 3 +- .../cpu/nativecpu/CpuNDArrayFactory.java | 5 +- .../aeron/ipc/AeronNDArraySubscriber.java | 2 +- .../nd4j/aeron/ipc/LargeNdArrayIpcTest.java | 36 +++++--- cavis-nd4j/cavis-nd4j-common/build.gradle | 2 + .../nd4j/common/io/ClassPathResourceTest.java | 7 +- .../build.gradle | 1 + .../client/ParameterServerClient.java | 8 +- .../RemoteParameterServerClientTests.java | 6 +- .../ParameterServerSubscriber.java | 2 +- .../zoo/model/TextGenerationLSTM.java | 2 +- nd4j/pom.xml | 35 ++++---- 115 files changed, 252 insertions(+), 351 deletions(-) diff --git a/brutex-extended-tests/build.gradle b/brutex-extended-tests/build.gradle index db9ea1cae..bd53f61bd 100644 --- a/brutex-extended-tests/build.gradle +++ b/brutex-extended-tests/build.gradle @@ -66,6 +66,8 @@ dependencies { implementation projects.cavisDnn.cavisDnnParallelwrapper implementation projects.cavisZoo.cavisZooModels + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" } test { diff --git a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java index e32f65a67..29e80ce99 100644 --- a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java @@ -73,6 +73,7 @@ import org.nd4j.linalg.ops.transforms.Transforms; import java.io.*; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; @@ -154,6 +155,9 @@ public class IntegrationTestRunner { evaluationClassesSeen = new HashMap<>(); } + public static void runTest(TestCase tc, Path testDir) throws Exception { + runTest(tc, testDir.toFile()); + } public static void runTest(TestCase tc, File testDir) throws Exception { BaseDL4JTest.skipUnlessIntegrationTests(); //Tests will ONLY be run if integration test profile is enabled. //This could alternatively be done via maven surefire configuration diff --git a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java index ebf4a9442..6b50f265e 100644 --- a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java @@ -28,18 +28,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.nio.file.Path; ////@Ignore("AB - 2019/05/27 - Integration tests need to be updated") public class IntegrationTestsDL4J extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 300_000L; - } - @TempDir - public File testDir; + public Path testDir; @AfterAll public static void afterClass(){ diff --git a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java index cc86b5cb3..5eb1ce856 100644 --- a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java @@ -30,12 +30,6 @@ import java.io.File; public class IntegrationTestsSameDiff extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 300_000L; - } - @TempDir public File testDir; diff --git a/cavis-common-platform/build.gradle b/cavis-common-platform/build.gradle index 05368a76b..a6202c6a8 100644 --- a/cavis-common-platform/build.gradle +++ b/cavis-common-platform/build.gradle @@ -65,6 +65,7 @@ dependencies { /*Logging*/ api 'org.slf4j:slf4j-api:1.7.30' + api 'org.slf4j:slf4j-simple:1.7.25' api "org.apache.logging.log4j:log4j-core:2.17.0" api "ch.qos.logback:logback-classic:1.2.3" diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java index 8be648fa0..ddbf34503 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java @@ -21,6 +21,7 @@ package org.datavec.api.transform.ndarray; import lombok.Data; +import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.transform.MathOp; import org.datavec.api.transform.metadata.ColumnMetaData; @@ -36,6 +37,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; @Data +@EqualsAndHashCode(callSuper = false) public class NDArrayColumnsMathOpTransform extends BaseColumnsMathOpTransform { public NDArrayColumnsMathOpTransform(@JsonProperty("newColumnName") String newColumnName, diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java index ca5b7921c..74a91332c 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java @@ -21,6 +21,7 @@ package org.datavec.api.transform.ndarray; import lombok.Data; +import lombok.EqualsAndHashCode; import org.datavec.api.transform.MathFunction; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; @@ -32,6 +33,7 @@ import org.nd4j.linalg.ops.transforms.Transforms; import com.fasterxml.jackson.annotation.JsonProperty; @Data +@EqualsAndHashCode(callSuper = false) public class NDArrayMathFunctionTransform extends BaseColumnTransform { //Can't guarantee that the writable won't be re-used, for example in different Spark ops on the same RDD diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java index 4f4dcc4c3..708625c8a 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java @@ -21,6 +21,7 @@ package org.datavec.api.transform.ndarray; import lombok.Data; +import lombok.EqualsAndHashCode; import org.datavec.api.transform.MathOp; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.metadata.NDArrayMetaData; @@ -33,6 +34,7 @@ import org.nd4j.linalg.ops.transforms.Transforms; import com.fasterxml.jackson.annotation.JsonProperty; @Data +@EqualsAndHashCode(callSuper = false) public class NDArrayScalarOpTransform extends BaseColumnTransform { private final MathOp mathOp; diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java index 19a03ce83..2d31b5c05 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java @@ -21,6 +21,7 @@ package org.datavec.api.transform.transform.string; import lombok.Data; +import lombok.EqualsAndHashCode; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import com.fasterxml.jackson.annotation.JsonProperty; @@ -31,6 +32,7 @@ import java.util.Collections; import java.util.List; @Data +@EqualsAndHashCode(callSuper = false) public class StringListToIndicesNDArrayTransform extends StringListToCountsNDArrayTransform { /** * @param columnName The name of the column to convert diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle index 49822b112..36788ea03 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle @@ -28,4 +28,5 @@ dependencies { implementation "commons-io:commons-io" testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" } \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java index 5f78f1611..dc75e7e1c 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java @@ -151,7 +151,7 @@ public class LFWLoader extends BaseImageLoader implements Serializable { } FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng); BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples, - numLabels, 0, batchSize, null); + numLabels, 0, batchSize, (String) null); inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest)); } diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java index cf3e4abbe..303ca742f 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java @@ -48,6 +48,11 @@ import static org.bytedeco.opencv.global.opencv_core.*; import static org.bytedeco.opencv.global.opencv_imgcodecs.*; import static org.bytedeco.opencv.global.opencv_imgproc.*; +/** + * Uses JavaCV to load images. Allowed formats: bmp, gif, jpg, jpeg, jp2, pbm, pgm, ppm, pnm, png, tif, tiff, exr, webp + * + * @author saudet + */ public class NativeImageLoader extends BaseImageLoader { private static final int MIN_BUFFER_STEP_SIZE = 64 * 1024; private byte[] buffer = null; @@ -57,14 +62,16 @@ public class NativeImageLoader extends BaseImageLoader { "png", "tif", "tiff", "exr", "webp", "BMP", "GIF", "JPG", "JPEG", "JP2", "PBM", "PGM", "PPM", "PNM", "PNG", "TIF", "TIFF", "EXR", "WEBP"}; - protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + protected OpenCVFrameConverter.ToMat converter; boolean direct = !Loader.getPlatform().startsWith("android"); /** * Loads images with no scaling or conversion. */ - public NativeImageLoader() {} + public NativeImageLoader() { + this.converter = new OpenCVFrameConverter.ToMat(); + } /** * Instantiate an image with the given @@ -74,6 +81,7 @@ public class NativeImageLoader extends BaseImageLoader { */ public NativeImageLoader(long height, long width) { + this(); this.height = height; this.width = width; } @@ -87,8 +95,7 @@ public class NativeImageLoader extends BaseImageLoader { * @param channels the number of channels for the image* */ public NativeImageLoader(long height, long width, long channels) { - this.height = height; - this.width = width; + this(height, width); this.channels = channels; } @@ -132,12 +139,9 @@ public class NativeImageLoader extends BaseImageLoader { } protected NativeImageLoader(NativeImageLoader other) { - this.height = other.height; - this.width = other.width; - this.channels = other.channels; + this(other.height, other.width, other.channels, other.multiPageMode); this.centerCropIfNeeded = other.centerCropIfNeeded; this.imageTransform = other.imageTransform; - this.multiPageMode = other.multiPageMode; } @Override @@ -252,8 +256,27 @@ public class NativeImageLoader extends BaseImageLoader { @Override public INDArray asMatrix(File f, boolean nchw) throws IOException { - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) { - return asMatrix(bis, nchw); + Mat mat = imread(f.getAbsolutePath(), IMREAD_ANYDEPTH | IMREAD_ANYCOLOR ); + INDArray a; + if (this.multiPageMode != null) { + a = asMatrix(mat.data(), mat.cols()); + }else{ + // Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); + if (mat == null || mat.empty()) { + PIX pix = pixReadMem(mat.data(), mat.cols()); + if (pix == null) { + throw new IOException("Could not decode image from input stream"); + } + mat = convert(pix); + pixDestroy(pix); + } + a = asMatrix(mat); + mat.deallocate(); + } + if(nchw) { + return a; + } else { + return a.permute(0, 2, 3, 1); //NCHW to NHWC } } @@ -264,6 +287,8 @@ public class NativeImageLoader extends BaseImageLoader { @Override public INDArray asMatrix(InputStream inputStream, boolean nchw) throws IOException { + throw new RuntimeException("not implemented"); + /* Mat mat = streamToMat(inputStream); INDArray a; if (this.multiPageMode != null) { @@ -286,6 +311,8 @@ public class NativeImageLoader extends BaseImageLoader { } else { return a.permute(0, 2, 3, 1); //NCHW to NHWC } + + */ } /** @@ -297,7 +324,7 @@ public class NativeImageLoader extends BaseImageLoader { private Mat streamToMat(InputStream is) throws IOException { if(buffer == null){ buffer = IOUtils.toByteArray(is); - if(buffer.length <= 0){ + if(buffer.length == 0){ throw new IOException("Could not decode image from input stream: input stream was empty (no data)"); } bufferMat = new Mat(buffer); @@ -354,9 +381,13 @@ public class NativeImageLoader extends BaseImageLoader { @Override public Image asImageMatrix(File f, boolean nchw) throws IOException { - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) { - return asImageMatrix(bis, nchw); - } + Mat image = imread(f.getAbsolutePath(), IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); + INDArray a = asMatrix(image); + if(!nchw) + a = a.permute(0,2,3,1); //NCHW to NHWC + Image i = new Image(a, image.channels(), image.rows(), image.cols()); + image.deallocate(); + return i; } @Override @@ -366,7 +397,8 @@ public class NativeImageLoader extends BaseImageLoader { @Override public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException { - Mat mat = streamToMat(inputStream); + throw new RuntimeException("Deprecated. Not implemented."); + /*Mat mat = streamToMat(inputStream); Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); if (image == null || image.empty()) { PIX pix = pixReadMem(mat.data(), mat.cols()); @@ -383,6 +415,8 @@ public class NativeImageLoader extends BaseImageLoader { image.deallocate(); return i; + + */ } /** @@ -545,10 +579,15 @@ public class NativeImageLoader extends BaseImageLoader { } public void asMatrixView(InputStream is, INDArray view) throws IOException { - Mat mat = streamToMat(is); - Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); + throw new RuntimeException("Not implemented"); + + } + + public void asMatrixView(String filename, INDArray view) throws IOException { + Mat image = imread(filename,IMREAD_ANYDEPTH | IMREAD_ANYCOLOR ); + //Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); if (image == null || image.empty()) { - PIX pix = pixReadMem(mat.data(), mat.cols()); + PIX pix = pixReadMem(image.data(), image.cols()); if (pix == null) { throw new IOException("Could not decode image from input stream"); } @@ -561,14 +600,8 @@ public class NativeImageLoader extends BaseImageLoader { image.deallocate(); } - public void asMatrixView(String filename, INDArray view) throws IOException { - asMatrixView(new File(filename), view); - } - public void asMatrixView(File f, INDArray view) throws IOException { - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) { - asMatrixView(bis, view); - } + asMatrixView(f.getAbsolutePath(), view); } public void asMatrixView(Mat image, INDArray view) throws IOException { diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java index 86a6a59c1..079049257 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java @@ -53,6 +53,10 @@ import java.io.*; import java.net.URI; import java.util.*; +/** +* Base class for the image record reader +* +*/ @Slf4j public abstract class BaseImageRecordReader extends BaseRecordReader { protected boolean finishedInputStreamSplit; @@ -344,7 +348,8 @@ public abstract class BaseImageRecordReader extends BaseRecordReader { ((NativeImageLoader) imageLoader).asMatrixView(currBatch.get(i), features.tensorAlongDimension(i, 1, 2, 3)); } catch (Exception e) { - System.out.println("Image file failed during load: " + currBatch.get(i).getAbsolutePath()); + System.out.println("Image file failed during load: " + currBatch.get(i).getAbsolutePath() + "\n" + e.getMessage()); + e.printStackTrace(); throw new RuntimeException(e); } } diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java index 76dc8a798..97755d82e 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java @@ -21,6 +21,7 @@ package org.datavec.image.transform; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; @@ -38,6 +39,7 @@ import org.bytedeco.opencv.opencv_core.*; @JsonIgnoreProperties({"borderValue"}) @JsonInclude(JsonInclude.Include.NON_NULL) @Data +@EqualsAndHashCode(callSuper = false) public class BoxImageTransform extends BaseImageTransform { private int width; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java index c1e00ce35..d9b827e22 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java @@ -21,6 +21,7 @@ package org.datavec.image.transform; import lombok.Data; +import lombok.EqualsAndHashCode; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; import com.fasterxml.jackson.annotation.JsonInclude; @@ -32,6 +33,7 @@ import static org.bytedeco.opencv.global.opencv_core.*; @JsonInclude(JsonInclude.Include.NON_NULL) @Data +@EqualsAndHashCode(callSuper = false) public class FlipImageTransform extends BaseImageTransform { /** diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java index 0a12ee7f7..d823e5736 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java @@ -21,6 +21,7 @@ package org.datavec.image.transform; import lombok.Data; +import lombok.EqualsAndHashCode; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; import org.nd4j.linalg.factory.Nd4j; @@ -32,6 +33,7 @@ import org.bytedeco.opencv.opencv_core.*; import static org.bytedeco.opencv.global.opencv_imgproc.*; @Data +@EqualsAndHashCode(callSuper = false) public class LargestBlobCropTransform extends BaseImageTransform { protected org.nd4j.linalg.api.rng.Random rng; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java index 9877d6ffc..72eb0883e 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java @@ -21,6 +21,7 @@ package org.datavec.image.transform; import lombok.Data; +import lombok.EqualsAndHashCode; import org.datavec.image.data.ImageWritable; import java.util.Random; @@ -28,6 +29,7 @@ import java.util.Random; import org.bytedeco.opencv.opencv_core.*; @Data +@EqualsAndHashCode(callSuper = false) public class MultiImageTransform extends BaseImageTransform { private PipelineImageTransform transform; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java index 84c4191be..524cf0a13 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java @@ -21,6 +21,7 @@ package org.datavec.image.transform; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NonNull; import org.datavec.image.data.ImageWritable; @@ -32,6 +33,7 @@ import java.util.*; import org.bytedeco.opencv.opencv_core.*; @Data +@EqualsAndHashCode(callSuper = false) public class PipelineImageTransform extends BaseImageTransform { protected List> imageTransforms; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java index 8d76ecb2c..86013c239 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java @@ -21,6 +21,7 @@ package org.datavec.image.transform; import lombok.Data; +import lombok.EqualsAndHashCode; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; import org.nd4j.linalg.factory.Nd4j; @@ -35,6 +36,7 @@ import org.bytedeco.opencv.opencv_core.*; @JsonIgnoreProperties({"rng", "converter"}) @JsonInclude(JsonInclude.Include.NON_NULL) @Data +@EqualsAndHashCode(callSuper = false) public class RandomCropTransform extends BaseImageTransform { protected int outputHeight; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/TestNativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/TestNativeImageLoader.java index 5bc82a570..9550867bb 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/TestNativeImageLoader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/TestNativeImageLoader.java @@ -612,28 +612,6 @@ public class TestNativeImageLoader { NativeImageLoader il = new NativeImageLoader(32, 32, 3); - //asMatrix(File, boolean) - INDArray a_nchw = il.asMatrix(f); - INDArray a_nchw2 = il.asMatrix(f, true); - INDArray a_nhwc = il.asMatrix(f, false); - - assertEquals(a_nchw, a_nchw2); - assertEquals(a_nchw, a_nhwc.permute(0,3,1,2)); - - - //asMatrix(InputStream, boolean) - try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ - a_nchw = il.asMatrix(is); - } - try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ - a_nchw2 = il.asMatrix(is, true); - } - try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ - a_nhwc = il.asMatrix(is, false); - } - assertEquals(a_nchw, a_nchw2); - assertEquals(a_nchw, a_nhwc.permute(0,3,1,2)); - //asImageMatrix(File, boolean) Image i_nchw = il.asImageMatrix(f); @@ -642,20 +620,6 @@ public class TestNativeImageLoader { assertEquals(i_nchw.getImage(), i_nchw2.getImage()); assertEquals(i_nchw.getImage(), i_nhwc.getImage().permute(0,3,1,2)); //NHWC to NCHW - - - //asImageMatrix(InputStream, boolean) - try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ - i_nchw = il.asImageMatrix(is); - } - try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ - i_nchw2 = il.asImageMatrix(is, true); - } - try(InputStream is = new BufferedInputStream(new FileInputStream(f))){ - i_nhwc = il.asImageMatrix(is, false); - } - assertEquals(i_nchw.getImage(), i_nchw2.getImage()); - assertEquals(i_nchw.getImage(), i_nhwc.getImage().permute(0,3,1,2)); //NHWC to NCHW } } diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java index 3ab872d91..c7920422f 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java @@ -20,11 +20,8 @@ package org.nd4j.autodiff.functions; -import lombok.Data; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import lombok.extern.slf4j.Slf4j; -import lombok.val; import onnx.Onnx; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/SDVariable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/SDVariable.java index bfbd49ff0..609a1ceba 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/SDVariable.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/SDVariable.java @@ -46,7 +46,9 @@ public class SDVariable implements Serializable { protected SameDiff sameDiff; @Getter + @Setter protected String varName; + @Getter @Setter protected VariableType variableType; @@ -83,18 +85,6 @@ public class SDVariable implements Serializable { return varName; } - public void setVarName(String varName) { - this.varName = varName; - } - - /** - * @deprecated Use {@link #name()} - */ - @Deprecated - public String getVarName(){ - return name(); - } - /** * Returns true if this variable is a place holder * @return diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java index 9f0bb48f8..dbc36b239 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java @@ -39,5 +39,6 @@ public class Variable { protected String outputOfOp; //Null for placeholders/constants. For array type SDVariables, the name of the op it's an output of protected List controlDeps; //Control dependencies: name of ops that must be available before this variable is considered available for execution protected SDVariable gradient; //Variable corresponding to the gradient of this variable + @Builder.Default protected int variableIndex = -1; } diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java index 560717ad8..a5e3f9b66 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java @@ -76,7 +76,7 @@ public class SameDiffUtils { public static ExternalErrorsFunction externalErrors(SameDiff sameDiff, Map externalGradients, SDVariable... inputs) { Preconditions.checkArgument(inputs != null && inputs.length > 0, "Require at least one SDVariable to" + - " be specified when using external errors: got %s", inputs); + " be specified when using external errors: got %s", (Object) inputs); ExternalErrorsFunction fn = new ExternalErrorsFunction(sameDiff, Arrays.asList(inputs), externalGradients); fn.outputVariable(); return fn; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java index e154fad61..670309928 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java @@ -49,7 +49,7 @@ import java.io.Serializable; import java.util.List; @Getter -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class EvaluationCalibration extends BaseEvaluation { public static final int DEFAULT_RELIABILITY_DIAG_NUM_BINS = 10; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java index afaf32b32..f3510d615 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java @@ -22,8 +22,10 @@ package org.nd4j.evaluation.curves; import lombok.Data; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.EqualsAndHashCode; @Data +@EqualsAndHashCode(callSuper = false) public class Histogram extends BaseHistogram { private final String title; private final double lower; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java index 24d0fe424..2c5cf41ac 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java @@ -21,6 +21,8 @@ package org.nd4j.linalg.api.memory.deallocation; import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.nd4j.linalg.api.memory.Deallocatable; import org.nd4j.linalg.api.memory.Deallocator; @@ -28,6 +30,7 @@ import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; @Data +@EqualsAndHashCode(callSuper = false) public class DeallocatableReference extends WeakReference { private String id; private Deallocator deallocator; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java index 7ae3ade6c..bcaa443d9 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; @@ -36,6 +37,7 @@ import java.util.List; @Slf4j @Data +@EqualsAndHashCode(callSuper = false) public abstract class BaseIndexAccumulation extends BaseOp implements IndexAccumulation { protected boolean keepDims = false; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java index 794b00eb0..36e6982a0 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java @@ -44,6 +44,9 @@ import java.lang.reflect.Array; import java.util.*; @Slf4j +@Builder +@AllArgsConstructor +@EqualsAndHashCode(callSuper = true) public class DynamicCustomOp extends DifferentialFunction implements CustomOp { private String opName; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java index 7c1fa9526..7981803e8 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.custom; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.val; import org.nd4j.autodiff.samediff.SDVariable; @@ -35,6 +36,7 @@ import java.util.List; @Data @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Flatten extends DynamicCustomOp { private int order; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java index b6664aeab..0e71c1db2 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java @@ -21,6 +21,8 @@ package org.nd4j.linalg.api.ops.impl.controlflow.compat; import java.util.List; + +import lombok.EqualsAndHashCode; import lombok.NonNull; import lombok.val; import org.nd4j.autodiff.samediff.SDVariable; @@ -39,6 +41,7 @@ import org.tensorflow.framework.NodeDef; import java.util.HashMap; import java.util.Map; +@EqualsAndHashCode(callSuper = false) public abstract class BaseCompatOp extends DynamicCustomOp { protected String frameName; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java index 2b24f6944..f88dd0f76 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.controlflow.compat; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; @@ -37,6 +38,7 @@ import java.util.Map; @Data @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Enter extends BaseCompatOp { protected boolean isConstant; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java index 518662ba8..b1f1b4b5c 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.controlflow.compat; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; @@ -36,6 +37,7 @@ import java.util.Map; @Data @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class While extends BaseCompatOp { protected boolean isConstant; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java index 8c7b7739b..fe08a1299 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.indexaccum; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.NonNull; import org.nd4j.autodiff.samediff.SDVariable; @@ -36,6 +37,7 @@ import java.util.List; @Data @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class FirstIndex extends BaseIndexAccumulation { protected Condition condition; protected double compare; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java index 7b0e47dce..305b859c7 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.indexaccum; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.NonNull; import org.nd4j.autodiff.samediff.SDVariable; @@ -38,6 +39,7 @@ import java.util.Map; @Data @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class LastIndex extends BaseIndexAccumulation { protected Condition condition; protected double compare; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java index 228996407..84da21d67 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.indexaccum.custom; import lombok.Data; +import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; @@ -38,6 +39,7 @@ import java.util.List; import java.util.Map; @Data +@EqualsAndHashCode(callSuper = false) public class ArgAmax extends DynamicCustomOp { protected boolean keepDims = false; private int[] dimensions; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java index 16db04b2c..38562baee 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.indexaccum.custom; import lombok.Data; +import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; @@ -38,6 +39,7 @@ import java.util.List; import java.util.Map; @Data +@EqualsAndHashCode(callSuper = false) public class ArgAmin extends DynamicCustomOp { protected boolean keepDims = false; private int[] dimensions; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java index e2a7438bd..557b1f21f 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.indexaccum.custom; import lombok.Data; +import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; @Data +@EqualsAndHashCode(callSuper = false) public class ArgMax extends DynamicCustomOp { protected boolean keepDims = false; private int[] dimensions; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java index 00445ee87..65d6ec1f3 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.api.ops.impl.indexaccum.custom; import lombok.Data; +import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; @Data +@EqualsAndHashCode(callSuper = false) public class ArgMin extends DynamicCustomOp { protected boolean keepDims = false; private int[] dimensions; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java index d86ed9f48..833aa5263 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java @@ -22,16 +22,15 @@ package org.nd4j.linalg.api.ops.impl.layers.convolution.config; import java.util.LinkedHashMap; import java.util.Map; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.NonNull; + +import lombok.*; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Conv1DConfig extends BaseConvolutionConfig { public static final String NCW = "NCW"; public static final String NWC = "NWC"; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java index 7981da12f..66e870027 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.common.base.Preconditions; import org.nd4j.enums.WeightsFormat; @@ -32,6 +33,7 @@ import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Conv2DConfig extends BaseConvolutionConfig { public static final String NCHW = "NCHW"; public static final String NHWC = "NHWC"; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java index 1607d6ccd..574868013 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java @@ -25,6 +25,7 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.util.ConvConfigUtil; @@ -32,6 +33,7 @@ import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Conv3DConfig extends BaseConvolutionConfig { public static final String NDHWC = "NDHWC"; public static final String NCDHW = "NCDHW"; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java index 0b57e5cd5..bf9a041b9 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.util.ConvConfigUtil; @@ -31,6 +32,7 @@ import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class DeConv2DConfig extends BaseConvolutionConfig { public static final String NCHW = "NCHW"; public static final String NHWC = "NHWC"; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java index 041cda3d2..6c793e7d0 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.util.ConvConfigUtil; @@ -31,6 +32,7 @@ import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class DeConv3DConfig extends BaseConvolutionConfig { public static final String NCDHW = "NCDHW"; public static final String NDHWC = "NDHWC"; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java index e7edfcf4e..400ddd733 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java @@ -24,12 +24,14 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class LocalResponseNormalizationConfig extends BaseConvolutionConfig { private double alpha, beta, bias; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java index 1bf19aad2..bab13ae06 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.linalg.api.ops.impl.layers.convolution.Pooling2D; import org.nd4j.linalg.api.ops.impl.layers.convolution.Pooling2D.Divisor; @@ -33,6 +34,7 @@ import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Pooling2DConfig extends BaseConvolutionConfig { @Builder.Default private long kH = -1, kW = -1; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java index 007c89e89..1ed04bcaf 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.Map; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.nd4j.linalg.api.ops.impl.layers.convolution.Pooling3D; import org.nd4j.linalg.api.ops.impl.layers.convolution.Pooling3D.Pooling3DType; @@ -32,6 +33,7 @@ import org.nd4j.linalg.util.ConvConfigUtil; @Data @Builder @NoArgsConstructor +@EqualsAndHashCode(callSuper = false) public class Pooling3DConfig extends BaseConvolutionConfig { @Builder.Default private long kD = -1, kW = -1, kH = -1; // kernel @Builder.Default private long sD = 1, sW = 1, sH = 1; // strides diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java index b37be57c0..be291a9c3 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java @@ -39,7 +39,7 @@ import org.tensorflow.framework.NodeDef; import java.lang.reflect.Field; import java.util.*; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class Mmul extends DynamicCustomOp { protected MMulTranspose mt; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java index 58ed6f027..4471244ef 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java @@ -32,7 +32,7 @@ import org.nd4j.common.util.ArrayUtil; import java.util.List; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class MmulBp extends DynamicCustomOp { protected MMulTranspose mt; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java index ea312036d..a7d63e4bd 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java @@ -32,7 +32,7 @@ import org.nd4j.linalg.factory.Nd4j; import java.util.*; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class BatchMmul extends DynamicCustomOp { protected int transposeA; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java index 0e400c756..b77a29eb4 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java @@ -39,10 +39,15 @@ import java.util.Map; public class BalanceMinibatches { private DataSetIterator dataSetIterator; private int numLabels; + @Builder.Default private Map> paths = Maps.newHashMap(); + @Builder.Default private int miniBatchSize = -1; + @Builder.Default private File rootDir = new File("minibatches"); + @Builder.Default private File rootSaveDir = new File("minibatchessave"); + @Builder.Default private List labelRootDirs = new ArrayList<>(); private DataNormalization dataNormalization; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java index 72785aba4..afb795a42 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java @@ -20,7 +20,8 @@ package org.nd4j.linalg.learning.config; -import lombok.*; +import lombok.Builder; +import lombok.Data; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.AdaMaxUpdater; import org.nd4j.linalg.learning.GradientUpdater; @@ -44,7 +45,8 @@ public class AdaMax implements IUpdater { public static final double DEFAULT_ADAMAX_BETA1_MEAN_DECAY = 0.9; public static final double DEFAULT_ADAMAX_BETA2_VAR_DECAY = 0.999; - @lombok.Builder.Default private double learningRate = DEFAULT_ADAMAX_LEARNING_RATE; // learning rate + @lombok.Builder.Default + private double learningRate = DEFAULT_ADAMAX_LEARNING_RATE; // learning rate private ISchedule learningRateSchedule; @lombok.Builder.Default private double beta1 = DEFAULT_ADAMAX_BETA1_MEAN_DECAY; // gradient moving avg decay rate @lombok.Builder.Default private double beta2 = DEFAULT_ADAMAX_BETA2_VAR_DECAY; // gradient sqrd decay rate diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java index a62ccd21c..067b25370 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java @@ -335,20 +335,6 @@ public class OpProfiler { } } - /** - * Dev-time method. - * - * @return - */ - protected StackAggregator getMixedOrderAggregator() { - // FIXME: remove this method, or make it protected - return mixedOrderAggregator; - } - - public StackAggregator getScalarAggregator() { - return scalarAggregator; - } - protected void updatePairs(String opName, String opClass) { // now we save pairs of ops/classes String cOpNameKey = prevOpName + " -> " + opName; diff --git a/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java index aca151aa2..0dcd1fe08 100644 --- a/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java +++ b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java @@ -58,13 +58,6 @@ public abstract class BaseDL4JTest { return DEFAULT_THREADS; } - /** - * Override this method to set the default timeout for methods in the test class - */ - public long getTimeoutMilliseconds(){ - return 90_000; - } - /** * Override this to set the profiling mode for the tests defined in the child class */ diff --git a/cavis-dnn/cavis-dnn-common/build.gradle b/cavis-dnn/cavis-dnn-common/build.gradle index e48cae638..4630d5ed5 100644 --- a/cavis-dnn/cavis-dnn-common/build.gradle +++ b/cavis-dnn/cavis-dnn-common/build.gradle @@ -16,4 +16,6 @@ dependencies { implementation 'org.apache.commons:commons-math3' implementation 'org.apache.commons:commons-lang3' implementation 'org.apache.commons:commons-compress' -} \ No newline at end of file + + testRuntimeOnly 'net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT' +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/TestDataSets.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/TestDataSets.java index b6978c969..b14583d8c 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/TestDataSets.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/TestDataSets.java @@ -27,11 +27,6 @@ import org.junit.jupiter.api.Test; public class TestDataSets extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 180000L; - } - @Test public void testTinyImageNetExists() throws Exception { //Simple sanity check on extracting diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java index f85c1fdf6..a0b73489a 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java @@ -35,11 +35,6 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; */ public class SvhnDataFetcherTest extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 480_000_000L; //Shouldn't take this long but slow download or drive access on CI machines may need extra time. - } - @Test public void testSvhnDataFetcher() throws Exception { assumeTrue(isIntegrationTests()); //Ignore unless integration tests - CI can get caught up on slow disk access diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java index d4d0e28a1..138298e89 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java @@ -59,11 +59,6 @@ import static org.junit.jupiter.api.Assertions.*; public class DataSetIteratorTest extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 360000; //Should run quickly; increased to large timeout due to occasonal slow CI downloads - } - @Test public void testBatchSizeOfOneIris() throws Exception { //Test for (a) iterators returning correct number of examples, and diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java index b4e790ea1..2774e9961 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java @@ -21,6 +21,7 @@ package org.deeplearning4j.earlystopping; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.TestUtils; @@ -817,6 +818,7 @@ public class TestEarlyStopping extends BaseDL4JTest { } @Data + @EqualsAndHashCode(callSuper = false) public static class TestListener extends BaseTrainingListener { private int countEpochStart = 0; private int countEpochEnd = 0; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java index f39be0929..739168b31 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java @@ -51,10 +51,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; ////@Ignore public class AttentionLayerTest extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } @Test public void testSelfAttentionLayer() { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java index 6eb8c4e25..3d945b27e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java @@ -62,11 +62,6 @@ public class BNGradientCheckTest extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testGradient2dSimple() { DataNormalization scaler = new NormalizerMinMaxScaler(); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java index cdd11b6f9..094034320 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java @@ -62,11 +62,6 @@ public class CNN1DGradientCheckTest extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 180000; - } - @Test public void testCnn1DWithLocallyConnected1D() { Nd4j.getRandom().setSeed(1337); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java index f7a9375f8..2c8f4dead 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java @@ -59,11 +59,6 @@ public class CNN3DGradientCheckTest extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testCnn3DPlain() { Nd4j.getRandom().setSeed(1337); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java index b3a9e1020..3772741d5 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java @@ -73,11 +73,6 @@ public class CNNGradientCheckTest extends BaseDL4JTest { return CNN2DFormat.values(); } - @Override - public long getTimeoutMilliseconds() { - return 999990000L; - } - @Test public void testGradientCNNMLN() { if(this.format != CNN2DFormat.NCHW) //Only test NCHW due to flat input format... diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java index acdd9be27..c0a6cad8e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java @@ -49,11 +49,6 @@ import java.util.Random; ////@Ignore public class CapsnetGradientCheckTest extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testCapsNet() { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/DropoutGradientCheck.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/DropoutGradientCheck.java index 7ca1064b3..193ede7ac 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/DropoutGradientCheck.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/DropoutGradientCheck.java @@ -59,11 +59,6 @@ public class DropoutGradientCheck extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testDropoutGradient() { int minibatch = 3; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GlobalPoolingGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GlobalPoolingGradientCheckTests.java index 214cb895e..f4b9d4dc5 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GlobalPoolingGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GlobalPoolingGradientCheckTests.java @@ -55,11 +55,6 @@ public class GlobalPoolingGradientCheckTests extends BaseDL4JTest { private static final double DEFAULT_MAX_REL_ERROR = 1e-3; private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testRNNGlobalPoolingBasicMultiLayer() { //Basic test of global pooling w/ LSTM diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTests.java index 17c43c5c0..cab80a69a 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTests.java @@ -70,11 +70,6 @@ public class GradientCheckTests extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testMinibatchApplication() { IrisDataSetIterator iter = new IrisDataSetIterator(30, 150); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsComputationGraph.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsComputationGraph.java index bd18f698b..be641898e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsComputationGraph.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsComputationGraph.java @@ -71,11 +71,6 @@ public class GradientCheckTestsComputationGraph extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 999999999L; - } - @Test public void testBasicIris() { Nd4j.getRandom().setSeed(12345); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsMasking.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsMasking.java index b5822bc3d..a444e1146 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsMasking.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsMasking.java @@ -59,11 +59,6 @@ public class GradientCheckTestsMasking extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - private static class GradientCheckSimpleScenario { private final ILossFunction lf; private final Activation act; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java index 3ab2efd59..ad1b564db 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java @@ -54,12 +54,6 @@ public class LRNGradientCheckTests extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test public void testGradientLRNSimple() { Nd4j.getRandom().setSeed(12345); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java index 00fef6150..452742f10 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java @@ -55,11 +55,6 @@ public class LSTMGradientCheckTests extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testLSTMBasicMultiLayer() { //Basic test of GravesLSTM layer diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java index bc85841e3..fe4c1eb3b 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java @@ -73,11 +73,6 @@ public class LossFunctionGradientCheck extends BaseDL4JTest { private static final double DEFAULT_MAX_REL_ERROR = 1e-5; private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void lossFunctionGradientCheck() { ILossFunction[] lossFunctions = new ILossFunction[] {new LossBinaryXENT(), new LossBinaryXENT(), diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java index c9e65579b..8acbf157e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java @@ -52,11 +52,6 @@ public class NoBiasGradientCheckTests extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testGradientNoBiasDenseOutput() { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java index 12a1340e2..f11daf9ec 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java @@ -52,11 +52,6 @@ public class OutputLayerGradientChecks extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testRnnLossLayer() { Nd4j.getRandom().setSeed(12345L); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java index d1cbd5955..4555904ca 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java @@ -55,11 +55,6 @@ public class RnnGradientChecks extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test ////@Ignore("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") public void testBidirectionalWrapper() { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java index 25d594d9a..670987c78 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java @@ -56,11 +56,6 @@ public class UtilLayerGradientChecks extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testMaskLayer() { Nd4j.getRandom().setSeed(12345); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java index ec9fdab25..92ddf8622 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java @@ -57,11 +57,6 @@ public class VaeGradientCheckTests extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testVaeAsMLP() { //Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java index 85e513076..105fcb284 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java @@ -72,11 +72,6 @@ public class YoloGradientCheckTests extends BaseDL4JTest { @TempDir public File testDir; - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testYoloOutputLayer() { int depthIn = 2; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java index 039c3e4e6..b3e625849 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java @@ -186,11 +186,6 @@ public class DTypeTests extends BaseDL4JTest { TensorFlowCnnToFeedForwardPreProcessor.class //Deprecated )); - @Override - public long getTimeoutMilliseconds() { - return 9999999L; - } - @AfterAll public static void after() { ImmutableSet info; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java index e73a6fea6..f88e76a17 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class CustomActivation extends BaseActivationFunction implements IActivation { @Override public INDArray getActivation(INDArray in, boolean training) { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java index e50868b7a..10ca617fe 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java @@ -93,11 +93,6 @@ public class BatchNormalizationTest extends BaseDL4JTest { public void doBefore() { } - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testDnnForwardPass() { int nOut = 10; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java index 1f1c632e9..da674ea7c 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.layers.samediff.testlayers; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.deeplearning4j.nn.conf.graph.GraphVertex; import org.deeplearning4j.nn.conf.layers.samediff.SDVertexParams; @@ -37,6 +38,7 @@ import java.util.Map; @NoArgsConstructor @Data +@EqualsAndHashCode(callSuper = false) public class SameDiffDenseVertex extends SameDiffVertex { private int nIn; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java index e10f3180b..c8e758feb 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.multilayer; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.TestUtils; @@ -1424,6 +1425,7 @@ public class MultiLayerTest extends BaseDL4JTest { } @Data + @EqualsAndHashCode(callSuper = false) public static class CheckModelsListener extends BaseTrainingListener { private Set> modelClasses = new HashSet<>(); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/EncodedGradientsAccumulatorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/EncodedGradientsAccumulatorTest.java index 44ce85710..23347d950 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/EncodedGradientsAccumulatorTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/EncodedGradientsAccumulatorTest.java @@ -39,11 +39,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j public class EncodedGradientsAccumulatorTest extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 1200000L; - } - /** * This test ensures, that memory amount assigned to buffer is enough for any number of updates * @throws Exception diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java index 5d713ca59..78dbb6d14 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java @@ -39,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.*; @Slf4j ////@Ignore("AB 2019/05/21 - Failing (stuck, causing timeouts) - Issue #7657") -@Timeout(120000L) +//@Timeout(120000L) public class SmartFancyBlockingQueueTest extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java index 9b94b1b2c..4c3760d95 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java @@ -47,11 +47,6 @@ import static org.junit.jupiter.api.Assertions.*; public class TestCheckpointListener extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @TempDir public File tempDir; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java index 48e610dfb..47430c8c3 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java @@ -67,11 +67,6 @@ public class TestListeners extends BaseDL4JTest { @TempDir public File tempDir; - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - @Test public void testSettingListenersUnsupervised() { //Pretrain layers should get copies of the listeners, in addition to the diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java index da6976b6a..985f347d8 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java @@ -60,11 +60,6 @@ public class RegressionTest060 extends BaseDL4JTest { return DataType.FLOAT; } - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Test public void regressionTestMLP1() throws Exception { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java index e2ef4b233..2a75e7994 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java @@ -61,11 +61,6 @@ public class RegressionTest071 extends BaseDL4JTest { return DataType.FLOAT; } - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Test public void regressionTestMLP1() throws Exception { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java index b2af73f06..6566f03fe 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java @@ -60,11 +60,6 @@ public class RegressionTest080 extends BaseDL4JTest { return DataType.FLOAT; } - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Test public void regressionTestMLP1() throws Exception { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java index d2b20bea3..acee54871 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java @@ -57,11 +57,6 @@ import static org.junit.jupiter.api.Assertions.*; @Slf4j public class RegressionTest100a extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Override public DataType getDataType(){ return DataType.FLOAT; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java index 8cca8472e..8df2f258b 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java @@ -54,11 +54,6 @@ import static org.junit.jupiter.api.Assertions.*; public class RegressionTest100b3 extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Override public DataType getDataType(){ return DataType.FLOAT; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java index 71c928d84..5b4270a4e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java @@ -73,11 +73,6 @@ import org.nd4j.common.resources.Resources; public class RegressionTest100b4 extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Override public DataType getDataType() { return DataType.FLOAT; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java index cbf45e56d..40df45924 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java @@ -60,11 +60,6 @@ public class RegressionTest100b6 extends BaseDL4JTest { return DataType.FLOAT; } - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Test public void testCustomLayer() throws Exception { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java index 9d66e9b5a..8ec311167 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java @@ -31,11 +31,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class TestDistributionDeserializer extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } - @Test public void testDistributionDeserializer() throws Exception { //Test current format: diff --git a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle index 32d9a5c2f..683436dab 100644 --- a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle @@ -25,4 +25,5 @@ dependencies { implementation "org.slf4j:slf4j-api" implementation "org.apache.commons:commons-lang3" + implementation "com.fasterxml.jackson.core:jackson-annotations" } \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java index 480cdf6d3..ab88f2aa5 100644 --- a/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.modelimport.keras.layers; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; import org.deeplearning4j.common.config.DL4JClassLoading; @@ -46,6 +47,7 @@ import java.util.List; @Slf4j @Data +@EqualsAndHashCode(callSuper = false) public class TFOpLayerImpl extends AbstractLayer { diff --git a/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java index 4e8fcb0f9..480267fb5 100644 --- a/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.modelimport.keras.layers.core; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer; @@ -39,6 +40,7 @@ import java.util.Map; */ @Slf4j @Data +@EqualsAndHashCode(callSuper = false) public class KerasMasking extends KerasLayer { private double maskingValue; diff --git a/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java index 5c1a31ad0..332b74e73 100644 --- a/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.modelimport.keras.layers.core; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.nn.conf.graph.ElementWiseVertex; import org.deeplearning4j.nn.conf.graph.MergeVertex; @@ -35,6 +36,7 @@ import java.util.Map; @Slf4j @Data +@EqualsAndHashCode(callSuper = false) public class KerasMerge extends KerasLayer { private final String LAYER_FIELD_MODE = "mode"; diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java index 417154cf2..e15f8b9fc 100644 --- a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java @@ -55,11 +55,13 @@ public class VPTree implements Serializable { private Node root; private String similarityFunction; @Getter + @Builder.Default private boolean invert = false; private transient ExecutorService executorService; @Getter + @Builder.Default private int workers = 1; - private AtomicInteger size = new AtomicInteger(0); + @Builder.Default private AtomicInteger size = new AtomicInteger(0); private transient ThreadLocal scalars = new ThreadLocal<>(); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java index b5f4a5107..74f60a603 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java @@ -21,11 +21,13 @@ package org.deeplearning4j.eval.curves; import lombok.Data; +import lombok.EqualsAndHashCode; import org.nd4j.evaluation.curves.BaseHistogram; import com.fasterxml.jackson.annotation.JsonProperty; @Deprecated @Data +@EqualsAndHashCode(callSuper = false) public class Histogram extends org.nd4j.evaluation.curves.Histogram { /** diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java index 8d8532e0b..3a2170bc3 100644 --- a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java @@ -76,8 +76,9 @@ import static com.google.common.base.Preconditions.checkArgument; @JsonIgnoreProperties({"stats", "listeners", "iterationCount", "rng", "lastExportedRDDId", "lastRDDExportPath", "trainingMasterUID"}) @EqualsAndHashCode(exclude = {"stats", "listeners", "iterationCount", "rng", "lastExportedRDDId", "lastRDDExportPath", - "trainingMasterUID"}) + "trainingMasterUID"}, callSuper = false) @Slf4j + public class ParameterAveragingTrainingMaster extends BaseTrainingMaster implements TrainingMaster { diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java index c55b3268a..fcd13d516 100644 --- a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java @@ -21,6 +21,7 @@ package org.deeplearning4j.spark.parameterserver.training; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -100,7 +101,7 @@ import java.util.concurrent.atomic.AtomicInteger; */ @Slf4j @Data - +@EqualsAndHashCode(callSuper = false) public class SharedTrainingMaster extends BaseTrainingMaster implements TrainingMaster { //Static counter/id fields used to determine which training master last set up the singleton param servers, etc diff --git a/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java index 7de40dbdb..668b4e25b 100644 --- a/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java +++ b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java @@ -21,6 +21,7 @@ package org.nd4j.linalg.cpu.nativecpu; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.nd4j.common.base.Preconditions; @@ -578,8 +579,8 @@ public class CpuNDArrayFactory extends BaseNativeNDArrayFactory { * @return the concatenate ndarrays */ @Override - public INDArray concat(int dimension, INDArray... toConcat) { - if (toConcat == null || toConcat.length == 0) + public INDArray concat(int dimension, @NonNull INDArray... toConcat) { + if (toConcat.length == 0) throw new ND4JIllegalStateException("Can't concatenate 0 arrays"); if (toConcat.length == 1) diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java index 204893e97..0b554093a 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java @@ -51,7 +51,7 @@ public class AeronNDArraySubscriber implements AutoCloseable { // Create a context, needed for client connection to media driver // A separate media driver process need to run prior to running this application private Aeron.Context ctx; - private AtomicBoolean running = new AtomicBoolean(true); + private AtomicBoolean running; private final AtomicBoolean init = new AtomicBoolean(false); private NDArrayCallback ndArrayCallback; diff --git a/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java index 85ec9f01b..de0f74356 100644 --- a/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java @@ -72,18 +72,23 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { INDArray arr = Nd4j.ones(length); AeronNDArrayPublisher publisher; ctx = new Aeron.Context() - .driverTimeoutMs(10000).availableImageHandler(AeronUtil::printAvailableImage) - .unavailableImageHandler(AeronUtil::printUnavailableImage) - .aeronDirectoryName(mediaDriver.aeronDirectoryName()) - .errorHandler(err -> err.printStackTrace()); + .driverTimeoutMs(10000) + .availableImageHandler(AeronUtil::printAvailableImage) + .unavailableImageHandler(AeronUtil::printUnavailableImage) + .aeronDirectoryName(mediaDriver.aeronDirectoryName()) + .errorHandler(err -> err.printStackTrace()); final AtomicBoolean running = new AtomicBoolean(true); Aeron aeron = Aeron.connect(ctx); int numSubscribers = 1; AeronNDArraySubscriber[] subscribers = new AeronNDArraySubscriber[numSubscribers]; for (int i = 0; i < numSubscribers; i++) { - AeronNDArraySubscriber subscriber = AeronNDArraySubscriber.builder().streamId(streamId).ctx(getContext()) - .channel(channel).aeron(aeron).running(running).ndArrayCallback(new NDArrayCallback() { + AeronNDArraySubscriber subscriber = AeronNDArraySubscriber.builder() + .streamId(streamId).ctx(getContext()) + .channel(channel) + .aeron(aeron) + .running(running) + .ndArrayCallback(new NDArrayCallback() { /** * A listener for ndarray message * @@ -110,7 +115,8 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { try { subscriber.launch(); } catch (Exception e) { - log.error("",e); + System.out.println(e.getMessage()); + e.printStackTrace(); } }); @@ -122,17 +128,23 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { Thread.sleep(1000); - publisher = AeronNDArrayPublisher.builder().publishRetryTimeOut(3000).streamId(streamId).channel(channel) - .aeron(aeron).build(); + publisher = AeronNDArrayPublisher.builder() + .publishRetryTimeOut(3000) + .streamId(streamId) + .channel(channel) + .aeron(aeron) + .build(); - for (int i = 0; i < 1 && running.get(); i++) { - log.info("About to send array."); + for (int i = 0; i < 1; i++) { + System.out.println("About to send array."); publisher.publish(arr); - log.info("Sent array"); + System.out.println("Sent array"); } + Thread.sleep( 5000); + for (int i = 0; i < numSubscribers; i++) CloseHelper.close(subscribers[i]); CloseHelper.close(aeron); diff --git a/cavis-nd4j/cavis-nd4j-common/build.gradle b/cavis-nd4j/cavis-nd4j-common/build.gradle index e801f3b5c..b58257047 100644 --- a/cavis-nd4j/cavis-nd4j-common/build.gradle +++ b/cavis-nd4j/cavis-nd4j-common/build.gradle @@ -18,6 +18,7 @@ * ***************************************************************************** * */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" dependencies { implementation "com.fasterxml.jackson.core:jackson-databind" @@ -29,4 +30,5 @@ dependencies { implementation "org.apache.commons:commons-compress" implementation "commons-codec:commons-codec" testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" } \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java index 9e1591f82..3e5f8bf72 100644 --- a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java @@ -24,9 +24,11 @@ package org.nd4j.common.io; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; import java.io.File; +import java.nio.file.Path; import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -36,13 +38,12 @@ public class ClassPathResourceTest { @Test - public void testDirExtractingIntelliJ() throws Exception { + public void testDirExtractingIntelliJ(@TempDir Path tempDir) throws Exception { //https://github.com/deeplearning4j/deeplearning4j/issues/6483 ClassPathResource cpr = new ClassPathResource("somedir"); - File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); - FileUtils.forceMkdir(f); + File f = tempDir.toFile(); cpr.copyDirectory(f); File[] files = f.listFiles(); diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle index af5e0aa84..99b6d1866 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle @@ -27,6 +27,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core" implementation "com.fasterxml.jackson.core:jackson-databind" implementation "org.slf4j:slf4j-api" + implementation "org.slf4j:slf4j-simple" implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerModel implementation projects.cavisNd4j.cavisNd4jAeron implementation projects.cavisDnn.cavisDnnApi diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java index b1b2d6abc..72bf20ecf 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java @@ -53,16 +53,16 @@ public class ParameterServerClient implements NDArrayCallback { //port to listen on for the subscriber private int subscriberPort; //the stream to listen on for the subscriber - private int subscriberStream = 11; + @Builder.Default private int subscriberStream = 11; //the "current" ndarray private AtomicReference arr; - private INDArray none = Nd4j.scalar(1.0); + @Builder.Default private INDArray none = Nd4j.scalar(1.0); private AtomicBoolean running; private String masterStatusHost; private int masterStatusPort; - private ObjectMapper objectMapper = new ObjectMapper(); + @Builder.Default private ObjectMapper objectMapper = new ObjectMapper(); private Aeron aeron; - private boolean compressArray = true; + @Builder.Default private boolean compressArray = true; /** * Tracks number of diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java index 62175a7fa..005443fe3 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java @@ -56,7 +56,11 @@ public class RemoteParameterServerClientTests extends BaseND4JTest { new MediaDriver.Context().threadingMode(ThreadingMode.DEDICATED).dirDeleteOnStart(true) .termBufferSparseFile(false).conductorIdleStrategy(new BusySpinIdleStrategy()) .receiverIdleStrategy(new BusySpinIdleStrategy()) - .senderIdleStrategy(new BusySpinIdleStrategy()); + .senderIdleStrategy(new BusySpinIdleStrategy()) + .driverTimeoutMs(1000*1000 *1000) + .clientLivenessTimeoutNs(1000*1000*1000) + .timerIntervalNs( 1000 * 1000); + mediaDriver = MediaDriver.launchEmbedded(ctx); aeron = Aeron.connect(getContext()); diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java index 6aaca2e49..59bce0ad0 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java @@ -325,7 +325,7 @@ public class ParameterServerSubscriber implements AutoCloseable { int tries=0; while (!subscriber.launched() && tries<12) { tries++; - LockSupport.parkNanos(100000); + Thread.sleep(2000); } if(!subscriber.launched()) { throw new Exception("Subscriber did not start in time."); diff --git a/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java index 9751db1a7..432c74231 100644 --- a/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java +++ b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java @@ -47,7 +47,7 @@ public class TextGenerationLSTM extends ZooModel { @Builder.Default private long seed = 1234; @Builder.Default private int maxLength = 40; @Builder.Default private int totalUniqueCharacters = 47; - private int[] inputShape = new int[] {maxLength, totalUniqueCharacters}; + @Builder.Default private int[] inputShape = new int[] {40, 47}; @Builder.Default private IUpdater updater = new RmsProp(0.01); @Builder.Default private CacheMode cacheMode = CacheMode.NONE; @Builder.Default private WorkspaceMode workspaceMode = WorkspaceMode.ENABLED; diff --git a/nd4j/pom.xml b/nd4j/pom.xml index 8dfe09d06..33f87c4b3 100644 --- a/nd4j/pom.xml +++ b/nd4j/pom.xml @@ -1,22 +1,23 @@