From acdd9c0a8a0b8d1ac18fa89c08bcd3f06a10ccd7 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 7 Oct 2022 10:49:08 +0200 Subject: [PATCH] More test fixes Signed-off-by: brian --- brutex-extended-tests/build.gradle | 2 + .../integration/IntegrationTestRunner.java | 4 ++ .../integration/IntegrationTestsDL4J.java | 8 +--- .../integration/IntegrationTestsSameDiff.java | 6 --- cavis-common-platform/build.gradle | 1 + .../image/loader/NativeImageLoader.java | 41 ++++++++++--------- .../recordreader/BaseImageRecordReader.java | 7 +++- .../java/org/deeplearning4j/BaseDL4JTest.java | 7 ---- .../earlystopping/TestEarlyStopping.java | 2 + .../custom/testclasses/CustomActivation.java | 2 +- .../testlayers/SameDiffDenseVertex.java | 2 + .../build.gradle | 1 + .../RemoteParameterServerClientTests.java | 6 ++- .../ParameterServerSubscriber.java | 2 +- 14 files changed, 49 insertions(+), 42 deletions(-) diff --git a/brutex-extended-tests/build.gradle b/brutex-extended-tests/build.gradle index db9ea1cae..bd53f61bd 100644 --- a/brutex-extended-tests/build.gradle +++ b/brutex-extended-tests/build.gradle @@ -66,6 +66,8 @@ dependencies { implementation projects.cavisDnn.cavisDnnParallelwrapper implementation projects.cavisZoo.cavisZooModels + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" } test { diff --git a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java index e32f65a67..29e80ce99 100644 --- a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java @@ -73,6 +73,7 @@ import org.nd4j.linalg.ops.transforms.Transforms; import java.io.*; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; @@ -154,6 +155,9 @@ public class IntegrationTestRunner { evaluationClassesSeen = new HashMap<>(); } + public static void runTest(TestCase tc, Path testDir) throws Exception { + runTest(tc, testDir.toFile()); + } public static void runTest(TestCase tc, File testDir) throws Exception { BaseDL4JTest.skipUnlessIntegrationTests(); //Tests will ONLY be run if integration test profile is enabled. //This could alternatively be done via maven surefire configuration diff --git a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java index ebf4a9442..6b50f265e 100644 --- a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java @@ -28,18 +28,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.nio.file.Path; ////@Ignore("AB - 2019/05/27 - Integration tests need to be updated") public class IntegrationTestsDL4J extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 300_000L; - } - @TempDir - public File testDir; + public Path testDir; @AfterAll public static void afterClass(){ diff --git a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java index cc86b5cb3..5eb1ce856 100644 --- a/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java @@ -30,12 +30,6 @@ import java.io.File; public class IntegrationTestsSameDiff extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 300_000L; - } - @TempDir public File testDir; diff --git a/cavis-common-platform/build.gradle b/cavis-common-platform/build.gradle index 05368a76b..a6202c6a8 100644 --- a/cavis-common-platform/build.gradle +++ b/cavis-common-platform/build.gradle @@ -65,6 +65,7 @@ dependencies { /*Logging*/ api 'org.slf4j:slf4j-api:1.7.30' + api 'org.slf4j:slf4j-simple:1.7.25' api "org.apache.logging.log4j:log4j-core:2.17.0" api "ch.qos.logback:logback-classic:1.2.3" diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java index cf3e4abbe..4db72decd 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java @@ -48,6 +48,11 @@ import static org.bytedeco.opencv.global.opencv_core.*; import static org.bytedeco.opencv.global.opencv_imgcodecs.*; import static org.bytedeco.opencv.global.opencv_imgproc.*; +/** + * Uses JavaCV to load images. Allowed formats: bmp, gif, jpg, jpeg, jp2, pbm, pgm, ppm, pnm, png, tif, tiff, exr, webp + * + * @author saudet + */ public class NativeImageLoader extends BaseImageLoader { private static final int MIN_BUFFER_STEP_SIZE = 64 * 1024; private byte[] buffer = null; @@ -57,14 +62,16 @@ public class NativeImageLoader extends BaseImageLoader { "png", "tif", "tiff", "exr", "webp", "BMP", "GIF", "JPG", "JPEG", "JP2", "PBM", "PGM", "PPM", "PNM", "PNG", "TIF", "TIFF", "EXR", "WEBP"}; - protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); + protected OpenCVFrameConverter.ToMat converter; boolean direct = !Loader.getPlatform().startsWith("android"); /** * Loads images with no scaling or conversion. */ - public NativeImageLoader() {} + public NativeImageLoader() { + this.converter = new OpenCVFrameConverter.ToMat(); + } /** * Instantiate an image with the given @@ -74,6 +81,7 @@ public class NativeImageLoader extends BaseImageLoader { */ public NativeImageLoader(long height, long width) { + this(); this.height = height; this.width = width; } @@ -87,8 +95,7 @@ public class NativeImageLoader extends BaseImageLoader { * @param channels the number of channels for the image* */ public NativeImageLoader(long height, long width, long channels) { - this.height = height; - this.width = width; + this(height, width); this.channels = channels; } @@ -132,12 +139,9 @@ public class NativeImageLoader extends BaseImageLoader { } protected NativeImageLoader(NativeImageLoader other) { - this.height = other.height; - this.width = other.width; - this.channels = other.channels; + this(other.height, other.width, other.channels, other.multiPageMode); this.centerCropIfNeeded = other.centerCropIfNeeded; this.imageTransform = other.imageTransform; - this.multiPageMode = other.multiPageMode; } @Override @@ -297,7 +301,7 @@ public class NativeImageLoader extends BaseImageLoader { private Mat streamToMat(InputStream is) throws IOException { if(buffer == null){ buffer = IOUtils.toByteArray(is); - if(buffer.length <= 0){ + if(buffer.length == 0){ throw new IOException("Could not decode image from input stream: input stream was empty (no data)"); } bufferMat = new Mat(buffer); @@ -545,10 +549,15 @@ public class NativeImageLoader extends BaseImageLoader { } public void asMatrixView(InputStream is, INDArray view) throws IOException { - Mat mat = streamToMat(is); - Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); + throw new RuntimeException("Not implemented"); + + } + + public void asMatrixView(String filename, INDArray view) throws IOException { + Mat image = imread(filename,IMREAD_ANYDEPTH | IMREAD_ANYCOLOR ); + //Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); if (image == null || image.empty()) { - PIX pix = pixReadMem(mat.data(), mat.cols()); + PIX pix = pixReadMem(image.data(), image.cols()); if (pix == null) { throw new IOException("Could not decode image from input stream"); } @@ -561,14 +570,8 @@ public class NativeImageLoader extends BaseImageLoader { image.deallocate(); } - public void asMatrixView(String filename, INDArray view) throws IOException { - asMatrixView(new File(filename), view); - } - public void asMatrixView(File f, INDArray view) throws IOException { - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f))) { - asMatrixView(bis, view); - } + asMatrixView(f.getAbsolutePath(), view); } public void asMatrixView(Mat image, INDArray view) throws IOException { diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java index 86a6a59c1..079049257 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java @@ -53,6 +53,10 @@ import java.io.*; import java.net.URI; import java.util.*; +/** +* Base class for the image record reader +* +*/ @Slf4j public abstract class BaseImageRecordReader extends BaseRecordReader { protected boolean finishedInputStreamSplit; @@ -344,7 +348,8 @@ public abstract class BaseImageRecordReader extends BaseRecordReader { ((NativeImageLoader) imageLoader).asMatrixView(currBatch.get(i), features.tensorAlongDimension(i, 1, 2, 3)); } catch (Exception e) { - System.out.println("Image file failed during load: " + currBatch.get(i).getAbsolutePath()); + System.out.println("Image file failed during load: " + currBatch.get(i).getAbsolutePath() + "\n" + e.getMessage()); + e.printStackTrace(); throw new RuntimeException(e); } } diff --git a/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java index aca151aa2..0dcd1fe08 100644 --- a/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java +++ b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java @@ -58,13 +58,6 @@ public abstract class BaseDL4JTest { return DEFAULT_THREADS; } - /** - * Override this method to set the default timeout for methods in the test class - */ - public long getTimeoutMilliseconds(){ - return 90_000; - } - /** * Override this to set the profiling mode for the tests defined in the child class */ diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java index b4e790ea1..2774e9961 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java @@ -21,6 +21,7 @@ package org.deeplearning4j.earlystopping; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.TestUtils; @@ -817,6 +818,7 @@ public class TestEarlyStopping extends BaseDL4JTest { } @Data + @EqualsAndHashCode(callSuper = false) public static class TestListener extends BaseTrainingListener { private int countEpochStart = 0; private int countEpochEnd = 0; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java index e73a6fea6..f88e76a17 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class CustomActivation extends BaseActivationFunction implements IActivation { @Override public INDArray getActivation(INDArray in, boolean training) { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java index 1f1c632e9..da674ea7c 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.layers.samediff.testlayers; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.deeplearning4j.nn.conf.graph.GraphVertex; import org.deeplearning4j.nn.conf.layers.samediff.SDVertexParams; @@ -37,6 +38,7 @@ import java.util.Map; @NoArgsConstructor @Data +@EqualsAndHashCode(callSuper = false) public class SameDiffDenseVertex extends SameDiffVertex { private int nIn; diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle index af5e0aa84..99b6d1866 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle @@ -27,6 +27,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core" implementation "com.fasterxml.jackson.core:jackson-databind" implementation "org.slf4j:slf4j-api" + implementation "org.slf4j:slf4j-simple" implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerModel implementation projects.cavisNd4j.cavisNd4jAeron implementation projects.cavisDnn.cavisDnnApi diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java index 62175a7fa..005443fe3 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java @@ -56,7 +56,11 @@ public class RemoteParameterServerClientTests extends BaseND4JTest { new MediaDriver.Context().threadingMode(ThreadingMode.DEDICATED).dirDeleteOnStart(true) .termBufferSparseFile(false).conductorIdleStrategy(new BusySpinIdleStrategy()) .receiverIdleStrategy(new BusySpinIdleStrategy()) - .senderIdleStrategy(new BusySpinIdleStrategy()); + .senderIdleStrategy(new BusySpinIdleStrategy()) + .driverTimeoutMs(1000*1000 *1000) + .clientLivenessTimeoutNs(1000*1000*1000) + .timerIntervalNs( 1000 * 1000); + mediaDriver = MediaDriver.launchEmbedded(ctx); aeron = Aeron.connect(getContext()); diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java index b32e97147..59bce0ad0 100644 --- a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java @@ -325,7 +325,7 @@ public class ParameterServerSubscriber implements AutoCloseable { int tries=0; while (!subscriber.launched() && tries<12) { tries++; - Thread.sleep(1000); + Thread.sleep(2000); } if(!subscriber.launched()) { throw new Exception("Subscriber did not start in time.");