Additional integration tests

Signed-off-by: brian <brian@brutex.de>
master
Brian Rosenberger 2022-10-04 20:07:17 +02:00
parent 2f8e6ea1f8
commit 7451979a3c
22 changed files with 13 additions and 229 deletions

View File

@ -37,6 +37,7 @@ dependencies {
implementation projects.cavisDatavec.cavisDatavecApi implementation projects.cavisDatavec.cavisDatavecApi
implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore
implementation projects.cavisDnn.cavisDnnCommon implementation projects.cavisDnn.cavisDnnCommon
implementation projects.cavisDnn.cavisDnnCommonTests
implementation projects.cavisDnn.cavisDnnApi implementation projects.cavisDnn.cavisDnnApi
implementation "org.slf4j:slf4j-api" implementation "org.slf4j:slf4j-api"
implementation "org.apache.hadoop:hadoop-client" implementation "org.apache.hadoop:hadoop-client"
@ -47,6 +48,8 @@ dependencies {
testImplementation "org.apache.spark:spark-sql_${scalaVersion}" testImplementation "org.apache.spark:spark-sql_${scalaVersion}"
testCompileOnly "org.scala-lang:scala-library" testCompileOnly "org.scala-lang:scala-library"
implementation "it.unimi.dsi:fastutil-core:8.5.8"
implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore
implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkParameterserver implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkParameterserver
implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore
@ -56,8 +59,13 @@ dependencies {
implementation projects.cavisUi.cavisUiModel implementation projects.cavisUi.cavisUiModel
implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore
implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators
implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage
implementation projects.cavisDnn.cavisDnnParallelwrapper
implementation projects.cavisZoo.cavisZooModels
} }
test { test {

View File

@ -21,6 +21,8 @@
package org.deeplearning4j.integration; package org.deeplearning4j.integration;
import com.google.common.collect.ImmutableSet;
import com.google.common.reflect.ClassPath;
import lombok.NonNull; import lombok.NonNull;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@ -44,13 +46,14 @@ import org.deeplearning4j.optimize.listeners.CollectScoresListener;
import org.deeplearning4j.parallelism.ParallelInference; import org.deeplearning4j.parallelism.ParallelInference;
import org.deeplearning4j.parallelism.inference.InferenceMode; import org.deeplearning4j.parallelism.inference.InferenceMode;
import org.deeplearning4j.util.ModelSerializer; import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.autodiff.listeners.records.History; import org.nd4j.autodiff.listeners.records.History;
import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.autodiff.samediff.VariableType; import org.nd4j.autodiff.samediff.VariableType;
import org.nd4j.autodiff.samediff.internal.SameDiffOp; import org.nd4j.autodiff.samediff.internal.SameDiffOp;
import org.nd4j.common.base.Preconditions; import org.nd4j.common.base.Preconditions;
import org.nd4j.common.primitives.Pair;
import org.nd4j.common.resources.Resources;
import org.nd4j.evaluation.IEvaluation; import org.nd4j.evaluation.IEvaluation;
import org.nd4j.evaluation.classification.*; import org.nd4j.evaluation.classification.*;
import org.nd4j.evaluation.regression.RegressionEvaluation; import org.nd4j.evaluation.regression.RegressionEvaluation;
@ -66,10 +69,6 @@ import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.BooleanIndexing; import org.nd4j.linalg.indexing.BooleanIndexing;
import org.nd4j.linalg.indexing.conditions.Conditions; import org.nd4j.linalg.indexing.conditions.Conditions;
import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.linalg.ops.transforms.Transforms;
import org.nd4j.common.primitives.Pair;
import org.nd4j.common.resources.Resources;
import com.google.common.collect.ImmutableSet;
import com.google.common.reflect.ClassPath;
import java.io.*; import java.io.*;
import java.lang.reflect.Modifier; import java.lang.reflect.Modifier;

View File

@ -63,7 +63,7 @@ import java.util.Map;
public class SameDiffRNNTestCases { public class SameDiffRNNTestCases {
public static TestCase getRnnCsvSequenceClassificationTestCase1() { public static TestCase getRnnCsvSequenceClassificationTestCase1() {
return new SameDiffRNNTestCases.RnnCsvSequenceClassificationTestCase1(); return new RnnCsvSequenceClassificationTestCase1();
} }
protected static class RnnCsvSequenceClassificationTestCase1 extends TestCase { protected static class RnnCsvSequenceClassificationTestCase1 extends TestCase {

View File

@ -1,106 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>net.brutex.ai</groupId>
<artifactId>deeplearning4j-parent</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dl4j-integration-tests</artifactId>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>net.brutex.ai</groupId>
<artifactId>nd4j-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.brutex.ai</groupId>
<artifactId>deeplearning4j-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.brutex.ai</groupId>
<artifactId>deeplearning4j-zoo</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.brutex.ai</groupId>
<artifactId>deeplearning4j-parallel-wrapper</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.brutex.ai</groupId>
<artifactId>deeplearning4j-common-tests</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.brutex.ai</groupId>
<artifactId>nd4j-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- Skip this module for installation -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
<executions>
<execution>
<id>default-install</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<!-- Skip this module for deployment -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -1,63 +0,0 @@
#DL4J and SameDiff Integration Tests
These tests are designed to check a number of aspects of DL4J and SameDiff:
1. Predictions (i.e., network output)
2. Training (training curves, parameters, gradient calculation)
3. Evaluation (accuracy, etc)
4. Model serialization (saving + loading models)
5. Overfitting sanity checks (make sure we can overfit a single example)
6. Data pipelines
7. Parallel Wrapper
8. Validating conditions that should always hold (frozen layer params don't change, for example)
They are designed for the following purposes:
1. Detecting regressions: i.e., new commit changed or broke previously working functionality
2. Detecting integration issues - i.e., issues that show up only when components are used together (but not in isolation in unit test)
3. Detecting significant differences between CPU and CUDA backends
4. Validating implementation via sanity checks on training - i.e., can we overfit a single example?
5. Checking networks and data pipelines on real-world scale data and nets
6. Operating as fully automated pre-release checks (replacing manual sanity checks)
## Main Classes
Explanation of the main classes:
* **IntegrationTestBaselineGenerator**: Run *manually* to generate and save "expected results" for comparing in the future.
Output goes to dl4j-test-resources, for saving/uploading.
* **IntegrationTestRunner**: Actually runs the tests, and compares the output/result to those generated by the baseline generator
* **TestCase**: integration tests extend this
* **testcases/\*.java**: the actual integration test definitions
* **IntegrationTestsDL4J**: entry point for running the DL4J integration tests
* **IntegrationTestsSameDiff**: entry point for running the SameDiff integration tests
## Types of Test Components
The integration tests are set up to be able to run multiple types of tests on each network configuration.
Networks may be pretrained (from model zoo) or randomly initialized (from specified configuration).
Specifically, test cases can be run with any subset of the following components to be tested, by setting TestCase.XYZ boolean options to true or false:
1. **testPredictions**: Testing output (predictions) on some specified data vs. saved/known good arrays
2. **testGradients**: Testing gradients on some specified data vs. saved/known good arrays
3. **testPretrain**: Test layerwise pretraining parameters and training curves
4. **testTrainingCurves**: Train, and check score vs. iteration
5. **testParamsPostTraining**: validate params match post training
6. **testEvaluation**: test the evaluation performance (post training, if 4 or 5 are true)
7. **testParallelInference**: validate that single net and parallel inference results match
8. **testOverfitting**: sanity check - try to overfit a single example
See TestCase.java for more details.
## Adding a New Integration Test
The process to add a new test is simple:
1. Add a method that creates and returns a TestCase object (example: testcases/MLPTestCases.getMLPMnist())
2. Add it as a unit test to IntegrationTests class (example: IntegrationTestsDL4J.testMLPMnist())
3. Run IntegrationTestBaselineGenerator with the new test case, to generate and save the "known good" results.
4. Run the new integration test to make sure it passes, on both CPU and CUDA backends
5. Commit the generated test resources from step 3 to dl4j-test-resources repo
Note that IntegrationTestBaselineGenerator assumes you have the dl4j-test-resources cloned parallel to the DL4J mono-repo.

View File

@ -1,54 +0,0 @@
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<configuration>
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>logs/application.log</file>
<encoder>
<pattern> %logger{15} - %message%n%xException{5}
</pattern>
</encoder>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern> %logger{15} - %message%n%xException{5}
</pattern>
</encoder>
</appender>
<logger name="org.apache.catalina.core" level="DEBUG" />
<logger name="org.springframework" level="DEBUG" />
<logger name="org.deeplearning4j" level="INFO" />
<logger name="org.datavec" level="INFO" />
<logger name="org.nd4j" level="INFO" />
<logger name="opennlp.uima.util" level="OFF" />
<logger name="org.apache.uima" level="OFF" />
<logger name="org.cleartk" level="OFF" />
<root level="ERROR">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>