Additional integration tests

Signed-off-by: brian <brian@brutex.de>

brutex-extended-tests/build.gradle

@@ -37,6 +37,7 @@ dependencies {
     implementation projects.cavisDatavec.cavisDatavecApi
     implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore
     implementation projects.cavisDnn.cavisDnnCommon
+    implementation projects.cavisDnn.cavisDnnCommonTests
     implementation projects.cavisDnn.cavisDnnApi
     implementation "org.slf4j:slf4j-api"
     implementation "org.apache.hadoop:hadoop-client"
@@ -47,6 +48,8 @@ dependencies {
     testImplementation "org.apache.spark:spark-sql_${scalaVersion}"
     testCompileOnly "org.scala-lang:scala-library"
 
+    implementation "it.unimi.dsi:fastutil-core:8.5.8"
+
     implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore
     implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkParameterserver
     implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore
@@ -56,8 +59,13 @@ dependencies {
     implementation projects.cavisUi.cavisUiModel
     implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore
     implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode
+    implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets
     implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators
+    implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators
     implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage
+    implementation projects.cavisDnn.cavisDnnParallelwrapper
+
+    implementation projects.cavisZoo.cavisZooModels
 }
 
 test {

brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java

@@ -21,6 +21,8 @@
 package org.deeplearning4j.integration;
 
 
+import com.google.common.collect.ImmutableSet;
+import com.google.common.reflect.ClassPath;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
@@ -44,13 +46,14 @@ import org.deeplearning4j.optimize.listeners.CollectScoresListener;
 import org.deeplearning4j.parallelism.ParallelInference;
 import org.deeplearning4j.parallelism.inference.InferenceMode;
 import org.deeplearning4j.util.ModelSerializer;
-
 import org.nd4j.autodiff.listeners.records.History;
 import org.nd4j.autodiff.samediff.SDVariable;
 import org.nd4j.autodiff.samediff.SameDiff;
 import org.nd4j.autodiff.samediff.VariableType;
 import org.nd4j.autodiff.samediff.internal.SameDiffOp;
 import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.primitives.Pair;
+import org.nd4j.common.resources.Resources;
 import org.nd4j.evaluation.IEvaluation;
 import org.nd4j.evaluation.classification.*;
 import org.nd4j.evaluation.regression.RegressionEvaluation;
@@ -66,10 +69,6 @@ import org.nd4j.linalg.factory.Nd4j;
 import org.nd4j.linalg.indexing.BooleanIndexing;
 import org.nd4j.linalg.indexing.conditions.Conditions;
 import org.nd4j.linalg.ops.transforms.Transforms;
-import org.nd4j.common.primitives.Pair;
-import org.nd4j.common.resources.Resources;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.reflect.ClassPath;
 
 import java.io.*;
 import java.lang.reflect.Modifier;

brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java

@@ -63,7 +63,7 @@ import java.util.Map;
 public class SameDiffRNNTestCases {
 
     public static TestCase getRnnCsvSequenceClassificationTestCase1() {
-        return new SameDiffRNNTestCases.RnnCsvSequenceClassificationTestCase1();
+        return new RnnCsvSequenceClassificationTestCase1();
     }
 
     protected static class RnnCsvSequenceClassificationTestCase1 extends TestCase {

deeplearning4j/dl4j-integration-tests/pom.xml

@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ /* ******************************************************************************
-  ~  *
-  ~  *
-  ~  * This program and the accompanying materials are made available under the
-  ~  * terms of the Apache License, Version 2.0 which is available at
-  ~  * https://www.apache.org/licenses/LICENSE-2.0.
-  ~  *
-  ~  *  See the NOTICE file distributed with this work for additional
-  ~  *  information regarding copyright ownership.
-  ~  * Unless required by applicable law or agreed to in writing, software
-  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-  ~  * License for the specific language governing permissions and limitations
-  ~  * under the License.
-  ~  *
-  ~  * SPDX-License-Identifier: Apache-2.0
-  ~  ******************************************************************************/
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <parent>
-        <groupId>net.brutex.ai</groupId>
-        <artifactId>deeplearning4j-parent</artifactId>
-        <version>1.0.0-SNAPSHOT</version>
-    </parent>
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>dl4j-integration-tests</artifactId>
-
-
-    <dependencies>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>nd4j-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>deeplearning4j-core</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>deeplearning4j-zoo</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>deeplearning4j-parallel-wrapper</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>deeplearning4j-common-tests</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>net.brutex.ai</groupId>
-            <artifactId>nd4j-common</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-    </dependencies>
-
-    <build>
-        <plugins>
-            <!-- Skip this module for installation -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-install-plugin</artifactId>
-                <version>2.5.2</version>
-                <executions>
-                    <execution>
-                        <id>default-install</id>
-                        <phase>none</phase>
-                    </execution>
-                </executions>
-            </plugin>
-            <!-- Skip this module for deployment -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-deploy-plugin</artifactId>
-                <configuration>
-                    <skip>true</skip>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-</project>

deeplearning4j/dl4j-integration-tests/readme.md

@@ -1,63 +0,0 @@
-
-#DL4J and SameDiff Integration Tests
-
-These tests are designed to check a number of aspects of DL4J and SameDiff:
-1. Predictions (i.e., network output)
-2. Training (training curves, parameters, gradient calculation)
-3. Evaluation (accuracy, etc)
-4. Model serialization (saving + loading models)
-5. Overfitting sanity checks (make sure we can overfit a single example)
-6. Data pipelines
-7. Parallel Wrapper
-8. Validating conditions that should always hold (frozen layer params don't change, for example)
-
-
-They are designed for the following purposes:
-1. Detecting regressions: i.e., new commit changed or broke previously working functionality
-2. Detecting integration issues - i.e., issues that show up only when components are used together (but not in isolation in unit test)
-3. Detecting significant differences between CPU and CUDA backends
-4. Validating implementation via sanity checks on training - i.e., can we overfit a single example?
-5. Checking networks and data pipelines on real-world scale data and nets
-6. Operating as fully automated pre-release checks (replacing manual sanity checks)
-
-## Main Classes
-
-Explanation of the main classes:
-* **IntegrationTestBaselineGenerator**: Run *manually* to generate and save "expected results" for comparing in the future.
-  Output goes to dl4j-test-resources, for saving/uploading.
-* **IntegrationTestRunner**: Actually runs the tests, and compares the output/result to those generated by the baseline generator
-* **TestCase**: integration tests extend this
-* **testcases/\*.java**: the actual integration test definitions
-* **IntegrationTestsDL4J**: entry point for running the DL4J integration tests
-* **IntegrationTestsSameDiff**: entry point for running the SameDiff integration tests
-
-## Types of Test Components
-
-The integration tests are set up to be able to run multiple types of tests on each network configuration.
-
-Networks may be pretrained (from model zoo) or randomly initialized (from specified configuration).
-
-Specifically, test cases can be run with any subset of the following components to be tested, by setting TestCase.XYZ boolean options to true or false:
-
-1. **testPredictions**: Testing output (predictions) on some specified data vs. saved/known good arrays
-2. **testGradients**: Testing gradients on some specified data vs. saved/known good arrays
-3. **testPretrain**: Test layerwise pretraining parameters and training curves
-4. **testTrainingCurves**: Train, and check score vs. iteration
-5. **testParamsPostTraining**: validate params match post training
-6. **testEvaluation**: test the evaluation performance (post training, if 4 or 5 are true)
-7. **testParallelInference**: validate that single net and parallel inference results match
-8. **testOverfitting**: sanity check - try to overfit a single example
-
-See TestCase.java for more details.
-
-
-## Adding a New Integration Test
-
-The process to add a new test is simple:
-1. Add a method that creates and returns a TestCase object (example: testcases/MLPTestCases.getMLPMnist())
-2. Add it as a unit test to IntegrationTests class (example: IntegrationTestsDL4J.testMLPMnist())
-3. Run IntegrationTestBaselineGenerator with the new test case, to generate and save the "known good" results.
-4. Run the new integration test to make sure it passes, on both CPU and CUDA backends
-5. Commit the generated test resources from step 3 to dl4j-test-resources repo
-
-Note that IntegrationTestBaselineGenerator assumes you have the dl4j-test-resources cloned parallel to the DL4J mono-repo.

deeplearning4j/dl4j-integration-tests/src/test/resources/logback-test.xml

@@ -1,54 +0,0 @@
-<!--
-  ~ /* ******************************************************************************
-  ~  *
-  ~  *
-  ~  * This program and the accompanying materials are made available under the
-  ~  * terms of the Apache License, Version 2.0 which is available at
-  ~  * https://www.apache.org/licenses/LICENSE-2.0.
-  ~  *
-  ~  *  See the NOTICE file distributed with this work for additional
-  ~  *  information regarding copyright ownership.
-  ~  * Unless required by applicable law or agreed to in writing, software
-  ~  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-  ~  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-  ~  * License for the specific language governing permissions and limitations
-  ~  * under the License.
-  ~  *
-  ~  * SPDX-License-Identifier: Apache-2.0
-  ~  ******************************************************************************/
-  -->
-
-<configuration>
-
-    <appender name="FILE" class="ch.qos.logback.core.FileAppender">
-        <file>logs/application.log</file>
-        <encoder>
-            <pattern> %logger{15} - %message%n%xException{5}
-            </pattern>
-        </encoder>
-    </appender>
-
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern> %logger{15} - %message%n%xException{5}
-            </pattern>
-        </encoder>
-    </appender>
-
-    <logger name="org.apache.catalina.core" level="DEBUG" />
-    <logger name="org.springframework" level="DEBUG" />
-    <logger name="org.deeplearning4j" level="INFO" />
-    <logger name="org.datavec" level="INFO" />
-    <logger name="org.nd4j" level="INFO" />
-    <logger name="opennlp.uima.util" level="OFF" />
-    <logger name="org.apache.uima" level="OFF" />
-    <logger name="org.cleartk" level="OFF" />
-
-
-
-    <root level="ERROR">
-        <appender-ref ref="STDOUT" />
-        <appender-ref ref="FILE" />
-    </root>
-
-</configuration>