diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java index 391139f32..5c48b4c18 100644 --- a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java @@ -96,6 +96,11 @@ public class TestGraphLocalExecution extends BaseDL4JTest { Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); } + @Override + public long getTimeoutMilliseconds() { + return 90000L; + } + @Test public void testLocalExecutionDataSources() throws Exception { @@ -204,7 +209,7 @@ public class TestGraphLocalExecution extends BaseDL4JTest { OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() .candidateGenerator(candidateGenerator).dataProvider(dataProvider) .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(ScoreFunctions.testSetLoss(true)) - .terminationConditions(new MaxTimeCondition(5, TimeUnit.SECONDS), + .terminationConditions(new MaxTimeCondition(30, TimeUnit.SECONDS), new MaxCandidatesCondition(3)) .build(); @@ -251,7 +256,7 @@ public class TestGraphLocalExecution extends BaseDL4JTest { .candidateGenerator(candidateGenerator) .dataProvider(new TestMdsDataProvider(1, 32)) .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(ScoreFunctions.testSetLoss(true)) - .terminationConditions(new MaxTimeCondition(5, TimeUnit.SECONDS), + .terminationConditions(new MaxTimeCondition(30, TimeUnit.SECONDS), new MaxCandidatesCondition(3)) .scoreFunction(ScoreFunctions.testSetAccuracy()) .build(); diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java index 7a8e23ac0..3ba918548 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java +++ b/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java @@ -72,11 +72,11 @@ public class LFWLoader extends BaseImageLoader implements Serializable { protected File fullDir; protected boolean useSubset = false; - InputSplit[] inputSplit; + protected InputSplit[] inputSplit; - public static Map lfwData = new HashMap<>(); - public static Map lfwLabel = new HashMap<>(); - public static Map lfwSubsetData = new HashMap<>(); + public Map lfwData = new HashMap<>(); + public Map lfwLabel = new HashMap<>(); + public Map lfwSubsetData = new HashMap<>(); public LFWLoader() { this(false); diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java index bf6d908e2..7afdb7ac0 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java +++ b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java @@ -45,15 +45,23 @@ import static org.junit.Assert.assertTrue; */ public class LoaderTests { + private static void ensureDataAvailable(){ + //Ensure test resources available by initializing CifarLoader and relying on auto download + boolean preProcessCifar = false; + int numExamples = 10; + int row = 28; + int col = 28; + int channels = 1; + for( boolean train : new boolean[]{true, false}){ + CifarLoader loader = new CifarLoader(row, col, channels, train, preProcessCifar); + loader.next(numExamples); + } + new LFWLoader(new long[] {250, 250, 3}, true).getRecordReader(1, 1, 1, new Random(42)).next(); + } + @Test public void testLfwReader() throws Exception { - String subDir = "lfw-a/lfw"; - File path = new File(FilenameUtils.concat(System.getProperty("user.home"), subDir)); - FileSplit fileSplit = new FileSplit(path, LFWLoader.ALLOWED_FORMATS, new Random(42)); - BalancedPathFilter pathFilter = new BalancedPathFilter(new Random(42), LFWLoader.LABEL_PATTERN, 1, 1, 1); - InputSplit[] inputSplit = fileSplit.sample(pathFilter, 1); - RecordReader rr = new ImageRecordReader(250, 250, 3, LFWLoader.LABEL_PATTERN); - rr.initialize(inputSplit[0]); + RecordReader rr = new LFWLoader(new long[] {250, 250, 3}, true).getRecordReader(1, 1, 1, new Random(42)); List exptedLabel = rr.getLabels(); RecordReader rr2 = new LFWLoader(new long[] {250, 250, 3}, true).getRecordReader(1, 1, 1, new Random(42)); @@ -63,6 +71,7 @@ public class LoaderTests { @Test public void testCifarLoader() { + ensureDataAvailable(); File dir = new File(FilenameUtils.concat(System.getProperty("user.home"), "cifar/cifar-10-batches-bin")); CifarLoader cifar = new CifarLoader(false, dir); assertTrue(dir.exists()); @@ -71,6 +80,7 @@ public class LoaderTests { @Test public void testCifarInputStream() throws Exception { + ensureDataAvailable(); // check train String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin"; String path = FilenameUtils.concat(System.getProperty("user.home"), subDir); diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/MiscOpValidation.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/MiscOpValidation.java index 06c64445b..3998bc184 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/MiscOpValidation.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/MiscOpValidation.java @@ -2122,4 +2122,26 @@ public class MiscOpValidation extends BaseOpValidation { assertNull(err); } + + @Test + public void testSeqMask(){ + INDArray arr = Nd4j.createFromArray(1,2,3); + INDArray maxLen = Nd4j.scalar(4); + + INDArray out = Nd4j.create(DataType.INT32, 3, 4); + out.assign(Integer.MAX_VALUE); + + Nd4j.exec(DynamicCustomOp.builder("sequence_mask") + .addInputs(arr, maxLen) + .addOutputs(out) + .build() + ); + + INDArray exp = Nd4j.createFromArray(new int[][]{ + {1, 0, 0, 0}, + {1, 1, 0, 0}, + {1, 1, 1, 0}}); + + assertEquals(exp, out); + } } diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/ShapeOpValidation.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/ShapeOpValidation.java index 795cef3f1..0cbe52479 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/ShapeOpValidation.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/ShapeOpValidation.java @@ -22,6 +22,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.math3.linear.LUDecomposition; +import org.junit.Ignore; import org.junit.Test; import org.nd4j.OpValidationSuite; import org.nd4j.autodiff.samediff.SDVariable; @@ -2498,4 +2499,30 @@ public class ShapeOpValidation extends BaseOpValidation { assertEquals(Nd4j.createFromArray(2, 2), out); } + + + @Test @Ignore //AB 2020/04/01 - https://github.com/eclipse/deeplearning4j/issues/8592 + public void testReshapeZeros(){ + int[][] shapes = new int[][]{{2,0}, {10,0}, {10, 0}, {2,0,0,10}, {10, 0}, {0, 0, 10}, {0,2,10}, {1,2,0}}; + int[][] reshape = new int[][]{{2,-1}, {2,0,-1}, {5,2,-1}, {2,0,-1}, {-1, 2, 0}, {2, -1, 0}, {2, 0, 0, 0, -1}, {2,0,-1}}; + int[][] expected = new int[][]{{2,0}, {2,0,5}, {5,2,0}, {2,0,10}, {5,2,0}, {2,5,0}, {2,0,0,0,10}, {2,0,1}}; + + for( int i=0; i