Add long tags for longer running tests, misc fixes for test failures

.github/workflows/build-deploy-windows.yml

@@ -16,7 +16,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   windows-x86_64:
     needs: pre-ci
-    runs-on: windows-2019
+    runs-on: windows-2016
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.8.0

contrib/codegen-tools/codegen/pom.xml

@@ -124,6 +124,15 @@
                             </sources>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>get-cpu-count</id>
+                        <goals>
+                            <goal>cpu-count</goal>
+                        </goals>
+                        <configuration>
+                            <cpuCount>cpu.core.count</cpuCount>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
 

datavec/datavec-api/pom.xml

@@ -105,6 +105,12 @@
             <artifactId>fastutil</artifactId>
             <version>${fastutil.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.nd4j</groupId>
+            <artifactId>nd4j-common-tests</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java

@@ -94,10 +94,10 @@ public class TestMultiOpReduce extends BaseND4JTest {
     public void testReducerInteger() {
 
         List<List<Writable>> inputs = new ArrayList<>();
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(0)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2)));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(0)));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1)));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2)));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2)));
 
         Map<ReduceOp, Double> exp = new LinkedHashMap<>();
         exp.put(ReduceOp.Min, 0.0);
@@ -136,9 +136,7 @@ public class TestMultiOpReduce extends BaseND4JTest {
 
 
     @Test
-    @Disabled
     public void testReduceString() {
-
         List<List<Writable>> inputs = new ArrayList<>();
         inputs.add(Arrays.asList(new Text("someKey"), new Text("1")));
         inputs.add(Arrays.asList(new Text("someKey"), new Text("2")));
@@ -168,7 +166,7 @@ public class TestMultiOpReduce extends BaseND4JTest {
             assertEquals(out.get(0), new Text("someKey"));
 
             String msg = op.toString();
-            assertEquals(msg, exp.get(op), out.get(1).toString());
+            assertEquals(exp.get(op), out.get(1).toString(),msg);
         }
     }
 
@@ -176,12 +174,12 @@ public class TestMultiOpReduce extends BaseND4JTest {
     public void testReduceIntegerIgnoreInvalidValues() {
 
         List<List<Writable>> inputs = new ArrayList<>();
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("0")));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("1")));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("ignore me")));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("also ignore me")));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("2")));
+        inputs.add(Arrays.asList(new Text("someKey"), new Text("0")));
+        inputs.add(Arrays.asList(new Text("someKey"), new Text("1")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2)));
+        inputs.add(Arrays.asList(new Text("someKey"), new Text("ignore me")));
+        inputs.add(Arrays.asList(new Text("someKey"), new Text("also ignore me")));
+        inputs.add(Arrays.asList(new Text("someKey"), new Text("2")));
 
 
         Map<ReduceOp, Double> exp = new LinkedHashMap<>();
@@ -243,16 +241,16 @@ public class TestMultiOpReduce extends BaseND4JTest {
     public void testCustomReductions() {
 
         List<List<Writable>> inputs = new ArrayList<>();
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1), new Text("zero"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("zero"),
                         new DoubleWritable(0)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2), new Text("one"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("one"),
                         new DoubleWritable(1)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(3), new Text("two"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("two"),
                         new DoubleWritable(2)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(4), new Text("three"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("three"),
                         new DoubleWritable(3)));
 
-        List<Writable> expected = Arrays.asList((Writable) new Text("someKey"), new IntWritable(10), new Text("one"),
+        List<Writable> expected = Arrays.asList(new Text("someKey"), new IntWritable(10), new Text("one"),
                         new DoubleWritable(1));
 
 
@@ -293,16 +291,16 @@ public class TestMultiOpReduce extends BaseND4JTest {
     public void testCustomReductionsWithCondition() {
 
         List<List<Writable>> inputs = new ArrayList<>();
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1), new Text("zero"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("zero"),
                 new DoubleWritable(0)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2), new Text("one"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("one"),
                 new DoubleWritable(1)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(3), new Text("two"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("two"),
                 new DoubleWritable(2)));
-        inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(4), new Text("three"),
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("three"),
                 new DoubleWritable(3)));
 
-        List<Writable> expected = Arrays.asList((Writable) new Text("someKey"), new IntWritable(10), new IntWritable(3),
+        List<Writable> expected = Arrays.asList(new Text("someKey"), new IntWritable(10), new IntWritable(3),
                 new DoubleWritable(1));
 
 
@@ -346,7 +344,7 @@ public class TestMultiOpReduce extends BaseND4JTest {
         public IAggregableReduceOp<Writable, List<Writable>> reduceOp() {
             //For testing: let's take the second value
             return new AggregableMultiOp<>(Collections
-                            .<IAggregableReduceOp<Writable, Writable>>singletonList(new AggregableSecond<Writable>()));
+                            .<IAggregableReduceOp<Writable, Writable>>singletonList(new AggregableSecond<>()));
         }
 
         @Override
@@ -488,12 +486,12 @@ public class TestMultiOpReduce extends BaseND4JTest {
                         .addColumnString("filterCol").addColumnString("textCol").build();
 
         List<List<Writable>> inputs = new ArrayList<>();
-        inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(1), new Text("a"), new Text("zero")));
-        inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(2), new Text("b"), new Text("one")));
-        inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(3), new Text("a"), new Text("two")));
-        inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(4), new Text("b"), new Text("three")));
-        inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(5), new Text("a"), new Text("three")));
-        inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(6), new Text("b"), new Text("three")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("a"), new Text("zero")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("b"), new Text("one")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("a"), new Text("two")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("b"), new Text("three")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(5), new Text("a"), new Text("three")));
+        inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(6), new Text("b"), new Text("three")));
 
         Condition condition = new StringColumnCondition("filterCol", ConditionOp.Equal, "a");
 

datavec/datavec-arrow/pom.xml

@@ -56,6 +56,12 @@
             <artifactId>arrow-format</artifactId>
             <version>${arrow.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.nd4j</groupId>
+            <artifactId>nd4j-common-tests</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

datavec/datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java

@@ -49,6 +49,7 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
 
 
     @Test
+    @Tag(TagNames.NEEDS_VERIFY)
     @Disabled
     public void testBasicIndexing() {
         Schema.Builder schema = new Schema.Builder();
@@ -82,8 +83,9 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
     }
 
     @Test
-    //not worried about this till after next release
+    @Tag(TagNames.NEEDS_VERIFY)
     @Disabled
+    //not worried about this till after next release
     public void testVariableLengthTS() {
         Schema.Builder schema = new Schema.Builder()
                 .addColumnString("str")
@@ -91,13 +93,13 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
                 .addColumnDouble("dbl");
 
         List<List<Writable>> firstSeq = Arrays.asList(
-                Arrays.<Writable>asList(new Text("00"),new IntWritable(0),new DoubleWritable(2.0)),
-                Arrays.<Writable>asList(new Text("01"),new IntWritable(1),new DoubleWritable(2.1)),
-                Arrays.<Writable>asList(new Text("02"),new IntWritable(2),new DoubleWritable(2.2)));
+                Arrays.asList(new Text("00"),new IntWritable(0),new DoubleWritable(2.0)),
+                Arrays.asList(new Text("01"),new IntWritable(1),new DoubleWritable(2.1)),
+                Arrays.asList(new Text("02"),new IntWritable(2),new DoubleWritable(2.2)));
 
         List<List<Writable>> secondSeq = Arrays.asList(
-                Arrays.<Writable>asList(new Text("10"),new IntWritable(10),new DoubleWritable(12.0)),
-                Arrays.<Writable>asList(new Text("11"),new IntWritable(11),new DoubleWritable(12.1)));
+                Arrays.asList(new Text("10"),new IntWritable(10),new DoubleWritable(12.0)),
+                Arrays.asList(new Text("11"),new IntWritable(11),new DoubleWritable(12.1)));
 
         List<List<List<Writable>>> sequences = Arrays.asList(firstSeq, secondSeq);
 

datavec/datavec-data/datavec-data-image/pom.xml

@@ -100,6 +100,17 @@
             <artifactId>hdf5-platform</artifactId>
             <version>${hdf5.version}-${javacpp-presets.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>ffmpeg-platform</artifactId>
+            <version>${ffmpeg.version}-${javacpp.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.nd4j</groupId>
+            <artifactId>nd4j-common-tests</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java

@@ -36,6 +36,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import org.junit.jupiter.api.DisplayName;
 import java.nio.file.Path;
+import java.util.UUID;
+
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.nd4j.common.tests.tags.NativeTag;
 import org.nd4j.common.tests.tags.TagNames;
@@ -48,11 +50,11 @@ class LabelGeneratorTest {
 
     @Test
     @DisplayName("Test Parent Path Label Generator")
-    @Disabled
     void testParentPathLabelGenerator(@TempDir Path testDir) throws Exception {
         File orig = new ClassPathResource("datavec-data-image/testimages/class0/0.jpg").getFile();
         for (String dirPrefix : new String[] { "m.", "m" }) {
-            File f = testDir.toFile();
+            File f = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile();
+            f.mkdirs();
             int numDirs = 3;
             int filesPerDir = 4;
             for (int i = 0; i < numDirs; i++) {

datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java

@@ -260,7 +260,6 @@ public class TestImageTransform {
         assertEquals(22, transformed[1], 0);
     }
 
-    @Disabled
     @Test
     public void testFilterImageTransform() throws Exception {
         ImageWritable writable = makeRandomImage(0, 0, 4);

datavec/datavec-excel/pom.xml

@@ -53,6 +53,12 @@
             <artifactId>poi-ooxml</artifactId>
             <version>${poi.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.nd4j</groupId>
+            <artifactId>nd4j-common-tests</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java

@@ -181,7 +181,7 @@ public class ExcelRecordReader extends FileRecordReader {
         List<Writable> ret = new ArrayList<>(currRow.getLastCellNum());
         for(Cell cell: currRow) {
             String cellValue = dataFormatter.formatCellValue(cell);
-            switch(cell.getCellTypeEnum()) {
+            switch(cell.getCellType()) {
                 case BLANK: ret.add(new Text("")); break;
                 case STRING: ret.add(new Text("")); break;
                 case BOOLEAN: ret.add(new BooleanWritable(Boolean.valueOf(cellValue))); break;

datavec/datavec-jdbc/datavecTests/README_DO_NOT_TOUCH_FILES.txt

@@ -0,0 +1,9 @@
+
+# *************************************************************************
+# ***              DO NOT TOUCH FILES IN THIS DIRECTORY!                ***
+# *** FILES IN THIS DIRECTORY AND SUBDIRECTORIES CONSTITUTE A DERBY     ***
+# *** DATABASE, WHICH INCLUDES THE DATA (USER AND SYSTEM) AND THE       ***
+# *** FILES NECESSARY FOR DATABASE RECOVERY.                            ***
+# *** EDITING, ADDING, OR DELETING ANY OF THESE FILES MAY CAUSE DATA    ***
+# *** CORRUPTION AND LEAVE THE DATABASE IN A NON-RECOVERABLE STATE.     ***
+# *************************************************************************

datavec/datavec-jdbc/datavecTests/log/README_DO_NOT_TOUCH_FILES.txt

@@ -0,0 +1,8 @@
+
+# *************************************************************************
+# ***              DO NOT TOUCH FILES IN THIS DIRECTORY!                ***
+# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE RECOVERY   ***
+# *** SYSTEM. EDITING, ADDING, OR DELETING FILES IN THIS DIRECTORY      ***
+# *** WILL CAUSE THE DERBY RECOVERY SYSTEM TO FAIL, LEADING TO          ***
+# *** NON-RECOVERABLE CORRUPT DATABASES.                                ***
+# *************************************************************************

datavec/datavec-jdbc/datavecTests/seg0/README_DO_NOT_TOUCH_FILES.txt

@@ -0,0 +1,8 @@
+
+# *************************************************************************
+# ***              DO NOT TOUCH FILES IN THIS DIRECTORY!                ***
+# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE TO STORE   *** 
+# *** USER AND SYSTEM DATA. EDITING, ADDING, OR DELETING FILES IN THIS  ***
+# *** DIRECTORY WILL CORRUPT THE ASSOCIATED DERBY DATABASE AND MAKE     ***
+# *** IT NON-RECOVERABLE.                                               ***
+# *************************************************************************