Timeouts and scala 2.12 for deeplearning4j-nlp-korean workaround (#380)

* Increase default timeout on Spark tests

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8840 disable deeplearning4j-nlp-korean module for scala 2.12

Signed-off-by: Alex Black <blacka101@gmail.com>

* Fix for change-scala-versions.sh

Signed-off-by: Alex Black <blacka101@gmail.com>

* CUDA test fixes + more timeout issues

Signed-off-by: Alex Black <blacka101@gmail.com>

* More CUDA

Signed-off-by: Alex Black <blacka101@gmail.com>

* Small fix for cuDNN subsampling + same mode

Signed-off-by: Alex Black <blacka101@gmail.com>

* Flaky test fix

Signed-off-by: Alex Black <blacka101@gmail.com>

* Reduce memory requirements for ValidateCuDNN BN test

Signed-off-by: Alex Black <blacka101@gmail.com>

* Fix slow/ineffirient scalnet tests

Signed-off-by: Alex Black <blacka101@gmail.com>

* Increase timeouts to avoid failures if CI machines are slower than expected

Signed-off-by: Alex Black <blacka101@gmail.com>

* Ignore flaky test (issue #8849) and increase timeout for slow CI downloads

Signed-off-by: Alex Black <blacka101@gmail.com>

change-scala-versions.sh

@@ -89,4 +89,14 @@ find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
 find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
   -exec bash -c "sed_i 's/\(scalaVersion>\)'$FROM_VERSION'<\/scalaVersion>/\1'$TO_VERSION'<\/scalaVersion>/g' {}" \;
   
+# Disable deeplearning4j-nlp-korean for scala 2.12 - see https://github.com/eclipse/deeplearning4j/issues/8840
+if [ $TO_VERSION = $SCALA_211_VERSION ]; then
+  #Enable
+  sed -i 's/        <!--<module>deeplearning4j-nlp-korean<\/module>-->/        <module>deeplearning4j-nlp-korean<\/module>/g' deeplearning4j/deeplearning4j-nlp-parent/pom.xml
+else
+  #Disable
+  sed -i 's/        <module>deeplearning4j-nlp-korean<\/module>/        <!--<module>deeplearning4j-nlp-korean<\/module>-->/g' deeplearning4j/deeplearning4j-nlp-parent/pom.xml
+fi
+
+
 echo "Done updating Scala versions.";

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/TestUtils.java

@@ -31,6 +31,7 @@ import org.deeplearning4j.nn.layers.recurrent.LSTM;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
 import org.deeplearning4j.util.ModelSerializer;
 import org.nd4j.base.Preconditions;
+import org.nd4j.linalg.api.buffer.DataType;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution;
 import org.nd4j.linalg.factory.Nd4j;
@@ -124,12 +125,20 @@ public class TestUtils {
         return randomOneHot(examples, nOut, new Random(12345));
     }
 
+    public static INDArray randomOneHot(DataType dataType, long examples, long nOut){
+        return randomOneHot(dataType, examples, nOut, new Random(12345));
+    }
+
     public static INDArray randomOneHot(long examples, long nOut, long rngSeed){
         return randomOneHot(examples, nOut, new Random(rngSeed));
     }
 
     public static INDArray randomOneHot(long examples, long nOut, Random rng) {
-        INDArray arr = Nd4j.create(examples, nOut);
+        return randomOneHot(Nd4j.defaultFloatingPointType(), examples,nOut, rng);
+    }
+
+    public static INDArray randomOneHot(DataType dataType, long examples, long nOut, Random rng){
+        INDArray arr = Nd4j.create(dataType, examples, nOut);
         for( int i=0; i<examples; i++ ){
             arr.putScalar(i, rng.nextInt((int) nOut), 1.0);
         }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java

@@ -59,7 +59,7 @@ public class DataSetIteratorTest extends BaseDL4JTest {
 
     @Override
     public long getTimeoutMilliseconds() {
-        return 90000;
+        return 360000;  //Should run quickly; increased to large timeout due to occasonal slow CI downloads
     }
 
     @Test

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java

@@ -46,6 +46,11 @@ public class AttentionLayerTest extends BaseDL4JTest {
     @Rule
     public ExpectedException exceptionRule = ExpectedException.none();
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testSelfAttentionLayer() {
         int nIn = 3;

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java

@@ -61,6 +61,11 @@ public class BNGradientCheckTest extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testGradient2dSimple() {
         DataNormalization scaler = new NormalizerMinMaxScaler();

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java

@@ -54,6 +54,11 @@ public class CNN1DGradientCheckTest extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testCnn1DWithLocallyConnected1D() {
         Nd4j.getRandom().setSeed(1337);

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java

@@ -55,6 +55,11 @@ public class CNN3DGradientCheckTest extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testCnn3DPlain() {
         Nd4j.getRandom().setSeed(1337);

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java

@@ -62,6 +62,11 @@ public class CNNGradientCheckTest extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testGradientCNNMLN() {
         //Parameterized test, testing combinations of:

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java

@@ -43,6 +43,11 @@ import java.util.Random;
 
 public class CapsnetGradientCheckTest extends BaseDL4JTest {
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testCapsNet() {
 

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/DropoutGradientCheck.java

@@ -58,6 +58,11 @@ public class DropoutGradientCheck extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testDropoutGradient() {
         int minibatch = 3;

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/GlobalPoolingGradientCheckTests.java

@@ -53,6 +53,11 @@ public class GlobalPoolingGradientCheckTests extends BaseDL4JTest {
     private static final double DEFAULT_MAX_REL_ERROR = 1e-3;
     private static final double DEFAULT_MIN_ABS_ERROR = 1e-8;
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testRNNGlobalPoolingBasicMultiLayer() {
         //Basic test of global pooling w/ LSTM

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTests.java

@@ -68,6 +68,11 @@ public class GradientCheckTests extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testMinibatchApplication() {
         IrisDataSetIterator iter = new IrisDataSetIterator(30, 150);

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/GradientCheckTestsComputationGraph.java

@@ -69,6 +69,11 @@ public class GradientCheckTestsComputationGraph extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testBasicIris() {
         Nd4j.getRandom().setSeed(12345);

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java

@@ -53,6 +53,11 @@ public class LRNGradientCheckTests extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
 
     @Test
     public void testGradientLRNSimple() {

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java

@@ -54,6 +54,11 @@ public class LSTMGradientCheckTests extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testLSTMBasicMultiLayer() {
         //Basic test of GravesLSTM layer

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java

@@ -73,6 +73,11 @@ public class LossFunctionGradientCheck extends BaseDL4JTest {
     private static final double DEFAULT_MAX_REL_ERROR = 1e-5;
     private static final double DEFAULT_MIN_ABS_ERROR = 1e-8;
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void lossFunctionGradientCheck() {
         ILossFunction[] lossFunctions = new ILossFunction[] {new LossBinaryXENT(), new LossBinaryXENT(),

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java

@@ -48,6 +48,11 @@ public class NoBiasGradientCheckTests extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testGradientNoBiasDenseOutput() {
 

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java

@@ -48,6 +48,11 @@ public class OutputLayerGradientChecks extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testRnnLossLayer() {
         Nd4j.getRandom().setSeed(12345L);

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java

@@ -52,6 +52,11 @@ public class RnnGradientChecks extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     @Ignore("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912")
     public void testBidirectionalWrapper() {

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java

@@ -52,6 +52,11 @@ public class UtilLayerGradientChecks extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testMaskLayer() {
         Nd4j.getRandom().setSeed(12345);

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java

@@ -56,6 +56,11 @@ public class VaeGradientCheckTests extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testVaeAsMLP() {
         //Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java

@@ -64,6 +64,11 @@ public class YoloGradientCheckTests extends BaseDL4JTest {
     @Rule
     public TemporaryFolder testDir = new TemporaryFolder();
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void testYoloOutputLayer() {
         int depthIn = 2;

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java

@@ -63,7 +63,8 @@ public class TransferLearningMLNTest extends BaseDL4JTest {
     public void simpleFineTune() {
 
         long rng = 12345L;
-        DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3));
+        Nd4j.getRandom().setSeed(rng);
+        DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3));
         //original conf
         NeuralNetConfiguration.Builder confToChange =
                         new NeuralNetConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.LBFGS)
@@ -123,7 +124,8 @@ public class TransferLearningMLNTest extends BaseDL4JTest {
 
     @Test
     public void testNoutChanges() {
-        DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 2));
+        Nd4j.getRandom().setSeed(12345);
+        DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT,10, 2));
 
         NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1));
         FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1))
@@ -185,7 +187,8 @@ public class TransferLearningMLNTest extends BaseDL4JTest {
 
     @Test
     public void testRemoveAndAdd() {
-        DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3));
+        Nd4j.getRandom().setSeed(12345);
+        DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT,10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3));
 
         NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1));
         FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build();
@@ -377,8 +380,9 @@ public class TransferLearningMLNTest extends BaseDL4JTest {
 
     @Test
     public void testAllWithCNN() {
+        Nd4j.getRandom().setSeed(12345);
 
-        DataSet randomData = new DataSet(Nd4j.rand(10, 28 * 28 * 3).reshape(10, 3, 28, 28), Nd4j.rand(10, 10));
+        DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 28 * 28 * 3).reshape(10, 3, 28, 28), TestUtils.randomOneHot(DataType.FLOAT,10, 10));
         MultiLayerNetwork modelToFineTune =
                         new MultiLayerNetwork(
                                         new NeuralNetConfiguration.Builder().seed(123)
@@ -528,8 +532,9 @@ public class TransferLearningMLNTest extends BaseDL4JTest {
 
     @Test
     public void testAllWithCNNNew() {
+        Nd4j.getRandom().setSeed(12345);
 
-        DataSet randomData = new DataSet(Nd4j.rand(10, 28 * 28 * 3).reshape(10, 3, 28, 28), Nd4j.rand(10, 10));
+        DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT,10, 28 * 28 * 3).reshape(10, 3, 28, 28), TestUtils.randomOneHot(10, 10));
         MultiLayerNetwork modelToFineTune =
                 new MultiLayerNetwork(
                         new NeuralNetConfiguration.Builder().seed(123)

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/TestDataTypes.java

@@ -39,6 +39,8 @@ import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.learning.config.Adam;
+import org.nd4j.linalg.learning.config.Sgd;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
 
 import java.lang.reflect.Field;
@@ -51,27 +53,9 @@ import static org.junit.Assert.*;
 @Slf4j
 public class TestDataTypes extends BaseDL4JTest {
 
-    private static DataType typeBefore;
-
-    @BeforeClass
-    public static void beforeClass(){
-        typeBefore = Nd4j.dataType();
-    }
-
-    @AfterClass
-    public static void afterClass(){
-        Nd4j.setDataType(typeBefore);
-    }
-
     @Override
-    public void beforeTest(){
+    public long getTimeoutMilliseconds() {
-        Nd4j.getExecutioner().setProfilingMode(getProfilingMode());
+        return 180000L;
-        Nd4j.setDataType(DataType.HALF);
-    }
-
-    @Override
-    public OpExecutioner.ProfilingMode getProfilingMode(){
-        return OpExecutioner.ProfilingMode.NAN_PANIC;
     }
 
     @Test
@@ -87,6 +71,7 @@ public class TestDataTypes extends BaseDL4JTest {
                 assertEquals(globalDtype, Nd4j.defaultFloatingPointType());
 
                 MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+                        .updater(new Sgd(1e-2))
                         .dataType(netDType)
                         .convolutionMode(ConvolutionMode.Same)
                         .activation(Activation.TANH)
@@ -140,11 +125,22 @@ public class TestDataTypes extends BaseDL4JTest {
         INDArray fp32Train = outMapTrain.get(DataType.FLOAT).castTo(DataType.DOUBLE);
         INDArray fp16Train = outMapTrain.get(DataType.HALF).castTo(DataType.DOUBLE);
 
-        assertTrue(fp64Train.equalsWithEps(fp32Train, 1e-3));
+        boolean eq64_32 = fp64Train.equalsWithEps(fp32Train, 1e-3);
-        assertTrue(fp64Train.equalsWithEps(fp16Train, 1e-2));
+        boolean eq64_16 = fp64Train.equalsWithEps(fp16Train, 1e-2);
-
 
+        if(!eq64_32){
+            System.out.println("FP64/32");
+            System.out.println("fp64Train:\n" + fp64Train);
+            System.out.println("fp32Train:\n" + fp32Train);
         }
 
-
+        if(!eq64_16){
+            System.out.println("FP64/16");
+            System.out.println("fp64Train:\n" + fp64Train);
+            System.out.println("fp16Train:\n" + fp16Train);
+        }
+
+        assertTrue(eq64_32);
+        assertTrue(eq64_16);
+    }
 }

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/ValidateCuDNN.java

@@ -52,6 +52,11 @@ import java.util.List;
 @Slf4j
 public class ValidateCuDNN extends BaseDL4JTest {
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 360000L;
+    }
+
     @Test
     public void validateConvLayers() {
         Nd4j.getRandom().setSeed(12345);
@@ -124,15 +129,16 @@ public class ValidateCuDNN extends BaseDL4JTest {
         validateLayers(net, classesToTest, true, fShape, lShape, CuDNNValidationUtil.MAX_REL_ERROR, CuDNNValidationUtil.MIN_ABS_ERROR);
     }
 
-    @Test @Ignore //AB 2019/05/21 - https://github.com/deeplearning4j/deeplearning4j/issues/7766
+    @Test
     public void validateConvLayersSimpleBN() {
         //Test ONLY BN - no other CuDNN functionality (i.e., DL4J impls for everything else)
         Nd4j.getRandom().setSeed(12345);
 
+        int minibatch = 8;
         int numClasses = 10;
         //imageHeight,imageWidth,channels
-        int imageHeight = 240;
+        int imageHeight = 48;
-        int imageWidth = 240;
+        int imageWidth = 48;
         int channels = 3;
         IActivation activation = new ActivationIdentity();
         MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder()
@@ -171,8 +177,8 @@ public class ValidateCuDNN extends BaseDL4JTest {
         MultiLayerNetwork net = new MultiLayerNetwork(multiLayerConfiguration);
         net.init();
 
-        int[] fShape = new int[]{32, channels, imageHeight, imageWidth};
+        int[] fShape = new int[]{minibatch, channels, imageHeight, imageWidth};
-        int[] lShape = new int[]{32, numClasses};
+        int[] lShape = new int[]{minibatch, numClasses};
 
         List<Class<?>> classesToTest = new ArrayList<>();
         classesToTest.add(org.deeplearning4j.nn.layers.normalization.BatchNormalization.class);
@@ -185,10 +191,11 @@ public class ValidateCuDNN extends BaseDL4JTest {
         //Test ONLY LRN - no other CuDNN functionality (i.e., DL4J impls for everything else)
         Nd4j.getRandom().setSeed(12345);
 
+        int minibatch = 8;
         int numClasses = 10;
         //imageHeight,imageWidth,channels
-        int imageHeight = 240;
+        int imageHeight = 48;
-        int imageWidth = 240;
+        int imageWidth = 48;
         int channels = 3;
         IActivation activation = new ActivationIdentity();
         MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder()
@@ -229,8 +236,8 @@ public class ValidateCuDNN extends BaseDL4JTest {
         MultiLayerNetwork net = new MultiLayerNetwork(multiLayerConfiguration);
         net.init();
 
-        int[] fShape = new int[]{32, channels, imageHeight, imageWidth};
+        int[] fShape = new int[]{minibatch, channels, imageHeight, imageWidth};
-        int[] lShape = new int[]{32, numClasses};
+        int[] lShape = new int[]{minibatch, numClasses};
 
         List<Class<?>> classesToTest = new ArrayList<>();
         classesToTest.add(org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization.class);

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/convolution/TestConvolution.java

@@ -70,6 +70,11 @@ public class TestConvolution extends BaseDL4JTest {
     @Rule
     public TemporaryFolder testDir = new TemporaryFolder();
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 240000L;
+    }
+
     @Test
     public void testSameModeActivationSizes() {
         int inH = 3;
@@ -117,6 +122,8 @@ public class TestConvolution extends BaseDL4JTest {
         for (ConvolutionMode c : cm) {
             for (ConvolutionLayer.AlgoMode a : new ConvolutionLayer.AlgoMode[]{ConvolutionLayer.AlgoMode.NO_WORKSPACE, ConvolutionLayer.AlgoMode.PREFER_FASTEST}) {
                 for (boolean conv : new boolean[]{true, false}) {
+                    String msg = c + " - " + a + " - " + (conv ? "conv" : "subsampling");
+                    System.out.println(msg);
 
                     org.deeplearning4j.nn.conf.layers.Layer l;
                     if (conv) {
@@ -125,7 +132,9 @@ public class TestConvolution extends BaseDL4JTest {
                         l = new SubsamplingLayer.Builder().kernelSize(4, 4).stride(2, 2).build();
                     }
 
-                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
+                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+                            .dataType(DataType.DOUBLE)
+                            .seed(12345)
                             .l2(0.0005).updater(new Sgd(0.01)).weightInit(WeightInit.XAVIER).convolutionMode(c).cudnnAlgoMode(a).list()
                             .layer(0, l)
                             .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
@@ -159,32 +168,32 @@ public class TestConvolution extends BaseDL4JTest {
                         throw new RuntimeException();
 
 
-                    INDArray in = Nd4j.rand(new int[]{1, 1, 20, 20}); //(20-4+0)/2 +1 = 9
+                    INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{1, 1, 20, 20}); //(20-4+0)/2 +1 = 9
 
                     INDArray outCudnn = layerCudnn.activate(in, false, LayerWorkspaceMgr.noWorkspaces());
                     INDArray outStd = layerStandard.activate(in, false, LayerWorkspaceMgr.noWorkspaces());
 
-                    assertEquals(outStd, outCudnn);
+                    assertEquals(msg, outStd, outCudnn);
 
 
                     //Check backprop:
-                    INDArray epsilon = Nd4j.rand(outStd.shape());
+                    INDArray epsilon = Nd4j.rand(DataType.DOUBLE, outStd.shape());
-                    Pair<Gradient, INDArray> pCudnn = layerCudnn.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
+                    Pair<Gradient, INDArray> pCudnn = layerCudnn.backpropGradient(epsilon.dup(), LayerWorkspaceMgr.noWorkspaces());
-                    Pair<Gradient, INDArray> pStd = layerStandard.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
+                    Pair<Gradient, INDArray> pStd = layerStandard.backpropGradient(epsilon.dup(), LayerWorkspaceMgr.noWorkspaces());
 
-                    System.out.println(Arrays.toString(pStd.getSecond().data().asFloat()));
+//                    System.out.println(Arrays.toString(pStd.getSecond().data().asFloat()));
-                    System.out.println(Arrays.toString(pCudnn.getSecond().data().asFloat()));
+//                    System.out.println(Arrays.toString(pCudnn.getSecond().data().asFloat()));
 
                     INDArray epsOutStd = pStd.getSecond();
                     INDArray epsOutCudnn = pCudnn.getSecond();
 
-                    assertTrue(epsOutStd.equalsWithEps(epsOutCudnn, 1e-4));
+                    assertTrue(msg, epsOutStd.equalsWithEps(epsOutCudnn, 1e-4));
 
                     if (conv) {
                         INDArray gradStd = pStd.getFirst().gradient();
                         INDArray gradCudnn = pCudnn.getFirst().gradient();
 
-                        assertTrue(gradStd.equalsWithEps(gradCudnn, 1e-4));
+                        assertTrue(msg, gradStd.equalsWithEps(gradCudnn, 1e-4));
                     }
                 }
             }
@@ -192,7 +201,7 @@ public class TestConvolution extends BaseDL4JTest {
     }
 
 
-    @Test @Ignore   //AB 2019/05/21 - Ignored to get master passing - issue logged here: https://github.com/deeplearning4j/deeplearning4j/issues/7766
+    @Test
     public void validateXceptionImport() throws Exception {
         File dir = testDir.newFolder();
         File fSource = Resources.asFile("modelimport/keras/examples/xception/xception_tf_keras_2.h5");

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java

@@ -61,6 +61,11 @@ public class CNNGradientCheckTest extends BaseDL4JTest {
         Nd4j.setDataType(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 180000L;
+    }
+
     @Test
     public void testGradientCNNMLN() {
         //Parameterized test, testing combinations of:

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/gradientcheck/CuDNNGradientChecks.java

@@ -77,6 +77,10 @@ public class CuDNNGradientChecks extends BaseDL4JTest {
         DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
     }
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 180000L;
+    }
 
     @Test
     public void testConvolutional() throws Exception {

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/lstm/ValidateCudnnDropout.java

@@ -30,6 +30,11 @@ import static org.junit.Assert.assertTrue;
 
 public class ValidateCudnnDropout extends BaseDL4JTest {
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 180000L;
+    }
+
     @Test
     public void testCudnnDropoutSimple() {
         for (int[] shape : new int[][]{{10, 10}, {5, 2, 5, 2}}) {

deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/lstm/ValidateCudnnLSTM.java

@@ -46,6 +46,11 @@ import static org.junit.Assert.*;
  */
 public class ValidateCudnnLSTM extends BaseDL4JTest {
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 180000L;
+    }
+
     @Test
     public void validateImplSimple() throws Exception {
 
@@ -109,7 +114,7 @@ public class ValidateCudnnLSTM extends BaseDL4JTest {
         mln1.computeGradientAndScore();
         mln2.computeGradientAndScore();
 
-        assertEquals(mln1.score(), mln2.score(), 1e-8);
+        assertEquals(mln1.score(), mln2.score(), 1e-5);
 
         Gradient g1 = mln1.gradient();
         Gradient g2 = mln2.gradient();

deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasModelEndToEndTest.java

@@ -98,7 +98,7 @@ public class KerasModelEndToEndTest extends BaseDL4JTest {
 
     @Override
     public long getTimeoutMilliseconds() {
-        return 90000L;
+        return 180000L;     //Most benchmarks should run very quickly; large timeout is to avoid issues with unusually slow download of test resources
     }
 
     @Test(expected = IllegalStateException.class)

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingLayer.java

@@ -116,10 +116,10 @@ public class SubsamplingLayer extends AbstractLayer<org.deeplearning4j.nn.conf.l
         int[] dilation = layerConf().getDilation();
 
         int[] pad;
-        int[] outSize = new int[]{(int)input.size(2), (int)input.size(3)};    //NCHW
+        int[] outSizeFwd = new int[]{(int)epsilon.size(2), (int)epsilon.size(3)};    //NCHW
         boolean same = convolutionMode == ConvolutionMode.Same;
         if (same) {
-            pad = ConvolutionUtils.getSameModeTopLeftPadding(outSize, new int[] {inH, inW}, kernel, strides, dilation);
+            pad = ConvolutionUtils.getSameModeTopLeftPadding(outSizeFwd, new int[] {inH, inW}, kernel, strides, dilation);
         } else {
             pad = layerConf().getPadding();
         }

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectorsTest.java

@@ -43,6 +43,12 @@ import static org.junit.Assert.assertNotEquals;
  * @author raver119@gmail.com
  */
 public class SparkSequenceVectorsTest extends BaseDL4JTest {
+
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 120000L;
+    }
+
     protected static List<Sequence<VocabWord>> sequencesCyclic;
     private JavaSparkContext sc;
 

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/word2vec/SparkWord2VecTest.java

@@ -48,6 +48,12 @@ import static org.junit.Assert.*;
  * @author raver119@gmail.com
  */
 public class SparkWord2VecTest extends BaseDL4JTest {
+
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 120000L;
+    }
+
     private static List<String> sentences;
     private JavaSparkContext sc;
 

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java

@@ -34,6 +34,11 @@ import java.util.Map;
 public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable {
     protected transient JavaSparkContext sc;
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 120000L;
+    }
+
     @Before
     public void before() throws Exception {
         sc = getContext();

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java

@@ -31,6 +31,7 @@ import org.deeplearning4j.spark.text.functions.CountCumSum;
 import org.deeplearning4j.spark.text.functions.TextPipeline;
 import org.deeplearning4j.text.stopwords.StopWords;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.primitives.Counter;
@@ -350,7 +351,7 @@ public class TextPipelineTest extends BaseSparkTest {
      *
      * @throws Exception
      */
-    @Test
+    @Test @Ignore   //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849
     public void testZipFunction1() throws Exception {
         JavaSparkContext sc = getContext();
         JavaRDD<String> corpusRDD = getCorpusRDD(sc);
@@ -388,7 +389,7 @@ public class TextPipelineTest extends BaseSparkTest {
         sc.stop();
     }
 
-    @Test
+    @Test @Ignore   //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849
     public void testZipFunction2() throws Exception {
         JavaSparkContext sc = getContext();
         JavaRDD<String> corpusRDD = getCorpusRDD(sc);

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java

@@ -53,6 +53,12 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable
     protected transient DataSet data;
     protected transient JavaRDD<DataSet> sparkData;
 
+
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 120000L;
+    }
+
     @Before
     public void before() {
 

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/BaseSparkKryoTest.java

@@ -28,6 +28,11 @@ import java.util.Map;
  */
 public class BaseSparkKryoTest extends BaseSparkTest {
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 120000L;
+    }
+
     @Override
     public JavaSparkContext getContext() {
         if (sc != null) {

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java

@@ -55,6 +55,11 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable
     protected transient DataSet data;
     protected transient JavaRDD<DataSet> sparkData;
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 120000L;
+    }
+
     @Before
     public void before() {
 

scalnet/src/test/scala/org/deeplearning4j/scalnet/examples/dl4j/feedforward/IrisCSVExample.scala

@@ -21,6 +21,7 @@ import org.datavec.api.records.reader.impl.csv.CSVRecordReader
 import org.datavec.api.split.FileSplit
 import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator
 import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator
+import org.deeplearning4j.nn.conf.Updater
 import org.deeplearning4j.optimize.listeners.ScoreIterationListener
 import org.deeplearning4j.scalnet.layers.core.Dense
 import org.deeplearning4j.scalnet.logging.Logging
@@ -29,6 +30,7 @@ import org.nd4j.linalg.activations.Activation
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator
 import org.nd4j.linalg.dataset.{ DataSet, SplitTestAndTrain }
 import org.nd4j.linalg.io.ClassPathResource
+import org.nd4j.linalg.learning.config.Adam
 import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction
 
 object IrisCSVExample extends App with Logging {
@@ -41,7 +43,7 @@ object IrisCSVExample extends App with Logging {
   val hiddenSize = 128
   val inputSize = 4
   val outputSize = 3
-  val epochs = 1000
+  val epochs = 20
   val scoreFrequency = 5
   val seed = 1234
 
@@ -64,7 +66,7 @@ object IrisCSVExample extends App with Logging {
   model.add(Dense(nOut = hiddenSize, activation = Activation.RELU))
   model.add(Dense(nOut = hiddenSize, activation = Activation.RELU))
   model.add(Dense(outputSize, activation = Activation.SOFTMAX))
-  model.compile(LossFunction.MCXENT)
+  model.compile(LossFunction.MCXENT, updater = Updater.ADAM)
 
   logger.info("Train model...")
   model.fit(training_data, epochs, List(new ScoreIterationListener(scoreFrequency)))

scalnet/src/test/scala/org/deeplearning4j/scalnet/examples/dl4j/recurrent/BasicRNNExample.scala

@@ -34,7 +34,7 @@ object BasicRNNExample extends App with Logging {
   val learningString = "*Der Cottbuser Postkutscher putzt den Cottbuser Postkutschkasten.".toVector
   val learningChars = learningString.distinct
   val hiddenSize = 64
-  val epochs = 200
+  val epochs = 20
   val seed = 1234
   val rand = new Random(seed)