parent
2f8e6ea1f8
commit
7451979a3c
|
|
@ -37,6 +37,7 @@ dependencies {
|
|||
implementation projects.cavisDatavec.cavisDatavecApi
|
||||
implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore
|
||||
implementation projects.cavisDnn.cavisDnnCommon
|
||||
implementation projects.cavisDnn.cavisDnnCommonTests
|
||||
implementation projects.cavisDnn.cavisDnnApi
|
||||
implementation "org.slf4j:slf4j-api"
|
||||
implementation "org.apache.hadoop:hadoop-client"
|
||||
|
|
@ -47,6 +48,8 @@ dependencies {
|
|||
testImplementation "org.apache.spark:spark-sql_${scalaVersion}"
|
||||
testCompileOnly "org.scala-lang:scala-library"
|
||||
|
||||
implementation "it.unimi.dsi:fastutil-core:8.5.8"
|
||||
|
||||
implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore
|
||||
implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkParameterserver
|
||||
implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore
|
||||
|
|
@ -56,8 +59,13 @@ dependencies {
|
|||
implementation projects.cavisUi.cavisUiModel
|
||||
implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore
|
||||
implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode
|
||||
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets
|
||||
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators
|
||||
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators
|
||||
implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage
|
||||
implementation projects.cavisDnn.cavisDnnParallelwrapper
|
||||
|
||||
implementation projects.cavisZoo.cavisZooModels
|
||||
}
|
||||
|
||||
test {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@
|
|||
package org.deeplearning4j.integration;
|
||||
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.reflect.ClassPath;
|
||||
import lombok.NonNull;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
|
@ -44,13 +46,14 @@ import org.deeplearning4j.optimize.listeners.CollectScoresListener;
|
|||
import org.deeplearning4j.parallelism.ParallelInference;
|
||||
import org.deeplearning4j.parallelism.inference.InferenceMode;
|
||||
import org.deeplearning4j.util.ModelSerializer;
|
||||
|
||||
import org.nd4j.autodiff.listeners.records.History;
|
||||
import org.nd4j.autodiff.samediff.SDVariable;
|
||||
import org.nd4j.autodiff.samediff.SameDiff;
|
||||
import org.nd4j.autodiff.samediff.VariableType;
|
||||
import org.nd4j.autodiff.samediff.internal.SameDiffOp;
|
||||
import org.nd4j.common.base.Preconditions;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.common.resources.Resources;
|
||||
import org.nd4j.evaluation.IEvaluation;
|
||||
import org.nd4j.evaluation.classification.*;
|
||||
import org.nd4j.evaluation.regression.RegressionEvaluation;
|
||||
|
|
@ -66,10 +69,6 @@ import org.nd4j.linalg.factory.Nd4j;
|
|||
import org.nd4j.linalg.indexing.BooleanIndexing;
|
||||
import org.nd4j.linalg.indexing.conditions.Conditions;
|
||||
import org.nd4j.linalg.ops.transforms.Transforms;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.common.resources.Resources;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.reflect.ClassPath;
|
||||
|
||||
import java.io.*;
|
||||
import java.lang.reflect.Modifier;
|
||||
|
|
@ -63,7 +63,7 @@ import java.util.Map;
|
|||
public class SameDiffRNNTestCases {
|
||||
|
||||
public static TestCase getRnnCsvSequenceClassificationTestCase1() {
|
||||
return new SameDiffRNNTestCases.RnnCsvSequenceClassificationTestCase1();
|
||||
return new RnnCsvSequenceClassificationTestCase1();
|
||||
}
|
||||
|
||||
protected static class RnnCsvSequenceClassificationTestCase1 extends TestCase {
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<parent>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>deeplearning4j-parent</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>dl4j-integration-tests</artifactId>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>nd4j-api</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>deeplearning4j-core</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>deeplearning4j-zoo</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>deeplearning4j-parallel-wrapper</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>deeplearning4j-common-tests</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.brutex.ai</groupId>
|
||||
<artifactId>nd4j-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- Skip this module for installation -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-install-plugin</artifactId>
|
||||
<version>2.5.2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>default-install</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!-- Skip this module for deployment -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
|
||||
#DL4J and SameDiff Integration Tests
|
||||
|
||||
These tests are designed to check a number of aspects of DL4J and SameDiff:
|
||||
1. Predictions (i.e., network output)
|
||||
2. Training (training curves, parameters, gradient calculation)
|
||||
3. Evaluation (accuracy, etc)
|
||||
4. Model serialization (saving + loading models)
|
||||
5. Overfitting sanity checks (make sure we can overfit a single example)
|
||||
6. Data pipelines
|
||||
7. Parallel Wrapper
|
||||
8. Validating conditions that should always hold (frozen layer params don't change, for example)
|
||||
|
||||
|
||||
They are designed for the following purposes:
|
||||
1. Detecting regressions: i.e., new commit changed or broke previously working functionality
|
||||
2. Detecting integration issues - i.e., issues that show up only when components are used together (but not in isolation in unit test)
|
||||
3. Detecting significant differences between CPU and CUDA backends
|
||||
4. Validating implementation via sanity checks on training - i.e., can we overfit a single example?
|
||||
5. Checking networks and data pipelines on real-world scale data and nets
|
||||
6. Operating as fully automated pre-release checks (replacing manual sanity checks)
|
||||
|
||||
## Main Classes
|
||||
|
||||
Explanation of the main classes:
|
||||
* **IntegrationTestBaselineGenerator**: Run *manually* to generate and save "expected results" for comparing in the future.
|
||||
Output goes to dl4j-test-resources, for saving/uploading.
|
||||
* **IntegrationTestRunner**: Actually runs the tests, and compares the output/result to those generated by the baseline generator
|
||||
* **TestCase**: integration tests extend this
|
||||
* **testcases/\*.java**: the actual integration test definitions
|
||||
* **IntegrationTestsDL4J**: entry point for running the DL4J integration tests
|
||||
* **IntegrationTestsSameDiff**: entry point for running the SameDiff integration tests
|
||||
|
||||
## Types of Test Components
|
||||
|
||||
The integration tests are set up to be able to run multiple types of tests on each network configuration.
|
||||
|
||||
Networks may be pretrained (from model zoo) or randomly initialized (from specified configuration).
|
||||
|
||||
Specifically, test cases can be run with any subset of the following components to be tested, by setting TestCase.XYZ boolean options to true or false:
|
||||
|
||||
1. **testPredictions**: Testing output (predictions) on some specified data vs. saved/known good arrays
|
||||
2. **testGradients**: Testing gradients on some specified data vs. saved/known good arrays
|
||||
3. **testPretrain**: Test layerwise pretraining parameters and training curves
|
||||
4. **testTrainingCurves**: Train, and check score vs. iteration
|
||||
5. **testParamsPostTraining**: validate params match post training
|
||||
6. **testEvaluation**: test the evaluation performance (post training, if 4 or 5 are true)
|
||||
7. **testParallelInference**: validate that single net and parallel inference results match
|
||||
8. **testOverfitting**: sanity check - try to overfit a single example
|
||||
|
||||
See TestCase.java for more details.
|
||||
|
||||
|
||||
## Adding a New Integration Test
|
||||
|
||||
The process to add a new test is simple:
|
||||
1. Add a method that creates and returns a TestCase object (example: testcases/MLPTestCases.getMLPMnist())
|
||||
2. Add it as a unit test to IntegrationTests class (example: IntegrationTestsDL4J.testMLPMnist())
|
||||
3. Run IntegrationTestBaselineGenerator with the new test case, to generate and save the "known good" results.
|
||||
4. Run the new integration test to make sure it passes, on both CPU and CUDA backends
|
||||
5. Commit the generated test resources from step 3 to dl4j-test-resources repo
|
||||
|
||||
Note that IntegrationTestBaselineGenerator assumes you have the dl4j-test-resources cloned parallel to the DL4J mono-repo.
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<configuration>
|
||||
|
||||
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
|
||||
<file>logs/application.log</file>
|
||||
<encoder>
|
||||
<pattern> %logger{15} - %message%n%xException{5}
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern> %logger{15} - %message%n%xException{5}
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="org.apache.catalina.core" level="DEBUG" />
|
||||
<logger name="org.springframework" level="DEBUG" />
|
||||
<logger name="org.deeplearning4j" level="INFO" />
|
||||
<logger name="org.datavec" level="INFO" />
|
||||
<logger name="org.nd4j" level="INFO" />
|
||||
<logger name="opennlp.uima.util" level="OFF" />
|
||||
<logger name="org.apache.uima" level="OFF" />
|
||||
<logger name="org.cleartk" level="OFF" />
|
||||
|
||||
|
||||
|
||||
<root level="ERROR">
|
||||
<appender-ref ref="STDOUT" />
|
||||
<appender-ref ref="FILE" />
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
Loading…
Reference in New Issue