From 7451979a3c7c0326b4a2a03743e6f62c85dcacba Mon Sep 17 00:00:00 2001 From: brian Date: Tue, 4 Oct 2022 20:07:17 +0200 Subject: [PATCH] Additional integration tests Signed-off-by: brian --- brutex-extended-tests/build.gradle | 8 ++ .../IntegrationTestBaselineGenerator.java | 0 .../integration/IntegrationTestRunner.java | 9 +- .../integration/IntegrationTestsDL4J.java | 0 .../integration/IntegrationTestsSameDiff.java | 0 .../deeplearning4j/integration/ModelType.java | 0 .../deeplearning4j/integration/TestCase.java | 0 .../deeplearning4j/integration/TestUtils.java | 0 .../testcases/dl4j/CNN1DTestCases.java | 0 .../testcases/dl4j/CNN2DTestCases.java | 0 .../testcases/dl4j/CNN3DTestCases.java | 0 .../testcases/dl4j/MLPTestCases.java | 0 .../testcases/dl4j/RNNTestCases.java | 0 .../testcases/dl4j/UnsupervisedTestCases.java | 0 .../dl4j/misc/CharacterIterator.java | 0 .../testcases/samediff/SameDiffCNNCases.java | 0 .../samediff/SameDiffMLPTestCases.java | 0 .../samediff/SameDiffRNNTestCases.java | 2 +- .../util/CountingMultiDataSetIterator.java | 0 deeplearning4j/dl4j-integration-tests/pom.xml | 106 ------------------ .../dl4j-integration-tests/readme.md | 63 ----------- .../src/test/resources/logback-test.xml | 54 --------- 22 files changed, 13 insertions(+), 229 deletions(-) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java (99%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/ModelType.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/TestCase.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/TestUtils.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN1DTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN2DTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN3DTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/MLPTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/UnsupervisedTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/dl4j/misc/CharacterIterator.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffCNNCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffMLPTestCases.java (100%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java (99%) rename {deeplearning4j/dl4j-integration-tests => brutex-extended-tests}/src/test/java/org/deeplearning4j/integration/util/CountingMultiDataSetIterator.java (100%) delete mode 100644 deeplearning4j/dl4j-integration-tests/pom.xml delete mode 100644 deeplearning4j/dl4j-integration-tests/readme.md delete mode 100644 deeplearning4j/dl4j-integration-tests/src/test/resources/logback-test.xml diff --git a/brutex-extended-tests/build.gradle b/brutex-extended-tests/build.gradle index d21da53de..db9ea1cae 100644 --- a/brutex-extended-tests/build.gradle +++ b/brutex-extended-tests/build.gradle @@ -37,6 +37,7 @@ dependencies { implementation projects.cavisDatavec.cavisDatavecApi implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnCommonTests implementation projects.cavisDnn.cavisDnnApi implementation "org.slf4j:slf4j-api" implementation "org.apache.hadoop:hadoop-client" @@ -47,6 +48,8 @@ dependencies { testImplementation "org.apache.spark:spark-sql_${scalaVersion}" testCompileOnly "org.scala-lang:scala-library" + implementation "it.unimi.dsi:fastutil-core:8.5.8" + implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkParameterserver implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore @@ -56,8 +59,13 @@ dependencies { implementation projects.cavisUi.cavisUiModel implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + implementation projects.cavisDnn.cavisDnnParallelwrapper + + implementation projects.cavisZoo.cavisZooModels } test { diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java similarity index 99% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java index 43c112d0a..e32f65a67 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java @@ -21,6 +21,8 @@ package org.deeplearning4j.integration; +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.ClassPath; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -44,13 +46,14 @@ import org.deeplearning4j.optimize.listeners.CollectScoresListener; import org.deeplearning4j.parallelism.ParallelInference; import org.deeplearning4j.parallelism.inference.InferenceMode; import org.deeplearning4j.util.ModelSerializer; - import org.nd4j.autodiff.listeners.records.History; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.autodiff.samediff.VariableType; import org.nd4j.autodiff.samediff.internal.SameDiffOp; import org.nd4j.common.base.Preconditions; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.resources.Resources; import org.nd4j.evaluation.IEvaluation; import org.nd4j.evaluation.classification.*; import org.nd4j.evaluation.regression.RegressionEvaluation; @@ -66,10 +69,6 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.BooleanIndexing; import org.nd4j.linalg.indexing.conditions.Conditions; import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.common.primitives.Pair; -import org.nd4j.common.resources.Resources; -import com.google.common.collect.ImmutableSet; -import com.google.common.reflect.ClassPath; import java.io.*; import java.lang.reflect.Modifier; diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/ModelType.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/ModelType.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/ModelType.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/ModelType.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/TestCase.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/TestCase.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/TestCase.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/TestCase.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/TestUtils.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/TestUtils.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/TestUtils.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/TestUtils.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN1DTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN1DTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN1DTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN1DTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN2DTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN2DTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN2DTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN2DTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN3DTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN3DTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN3DTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/CNN3DTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/MLPTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/MLPTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/MLPTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/MLPTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/UnsupervisedTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/UnsupervisedTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/UnsupervisedTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/UnsupervisedTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/misc/CharacterIterator.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/misc/CharacterIterator.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/misc/CharacterIterator.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/misc/CharacterIterator.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffCNNCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffCNNCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffCNNCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffCNNCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffMLPTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffMLPTestCases.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffMLPTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffMLPTestCases.java diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java similarity index 99% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java index e3daa2126..579aff0a4 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java +++ b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java @@ -63,7 +63,7 @@ import java.util.Map; public class SameDiffRNNTestCases { public static TestCase getRnnCsvSequenceClassificationTestCase1() { - return new SameDiffRNNTestCases.RnnCsvSequenceClassificationTestCase1(); + return new RnnCsvSequenceClassificationTestCase1(); } protected static class RnnCsvSequenceClassificationTestCase1 extends TestCase { diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/util/CountingMultiDataSetIterator.java b/brutex-extended-tests/src/test/java/org/deeplearning4j/integration/util/CountingMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/util/CountingMultiDataSetIterator.java rename to brutex-extended-tests/src/test/java/org/deeplearning4j/integration/util/CountingMultiDataSetIterator.java diff --git a/deeplearning4j/dl4j-integration-tests/pom.xml b/deeplearning4j/dl4j-integration-tests/pom.xml deleted file mode 100644 index bc595b5e7..000000000 --- a/deeplearning4j/dl4j-integration-tests/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - - net.brutex.ai - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - 4.0.0 - - dl4j-integration-tests - - - - - org.slf4j - slf4j-api - - - net.brutex.ai - nd4j-api - ${project.version} - - - net.brutex.ai - deeplearning4j-core - ${project.version} - - - net.brutex.ai - deeplearning4j-zoo - ${project.version} - - - net.brutex.ai - deeplearning4j-parallel-wrapper - ${project.version} - - - ch.qos.logback - logback-classic - test - - - net.brutex.ai - deeplearning4j-common-tests - ${project.version} - test - - - net.brutex.ai - nd4j-common - ${project.version} - test - - - - - - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - default-install - none - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - \ No newline at end of file diff --git a/deeplearning4j/dl4j-integration-tests/readme.md b/deeplearning4j/dl4j-integration-tests/readme.md deleted file mode 100644 index e0b9697e1..000000000 --- a/deeplearning4j/dl4j-integration-tests/readme.md +++ /dev/null @@ -1,63 +0,0 @@ - -#DL4J and SameDiff Integration Tests - -These tests are designed to check a number of aspects of DL4J and SameDiff: -1. Predictions (i.e., network output) -2. Training (training curves, parameters, gradient calculation) -3. Evaluation (accuracy, etc) -4. Model serialization (saving + loading models) -5. Overfitting sanity checks (make sure we can overfit a single example) -6. Data pipelines -7. Parallel Wrapper -8. Validating conditions that should always hold (frozen layer params don't change, for example) - - -They are designed for the following purposes: -1. Detecting regressions: i.e., new commit changed or broke previously working functionality -2. Detecting integration issues - i.e., issues that show up only when components are used together (but not in isolation in unit test) -3. Detecting significant differences between CPU and CUDA backends -4. Validating implementation via sanity checks on training - i.e., can we overfit a single example? -5. Checking networks and data pipelines on real-world scale data and nets -6. Operating as fully automated pre-release checks (replacing manual sanity checks) - -## Main Classes - -Explanation of the main classes: -* **IntegrationTestBaselineGenerator**: Run *manually* to generate and save "expected results" for comparing in the future. - Output goes to dl4j-test-resources, for saving/uploading. -* **IntegrationTestRunner**: Actually runs the tests, and compares the output/result to those generated by the baseline generator -* **TestCase**: integration tests extend this -* **testcases/\*.java**: the actual integration test definitions -* **IntegrationTestsDL4J**: entry point for running the DL4J integration tests -* **IntegrationTestsSameDiff**: entry point for running the SameDiff integration tests - -## Types of Test Components - -The integration tests are set up to be able to run multiple types of tests on each network configuration. - -Networks may be pretrained (from model zoo) or randomly initialized (from specified configuration). - -Specifically, test cases can be run with any subset of the following components to be tested, by setting TestCase.XYZ boolean options to true or false: - -1. **testPredictions**: Testing output (predictions) on some specified data vs. saved/known good arrays -2. **testGradients**: Testing gradients on some specified data vs. saved/known good arrays -3. **testPretrain**: Test layerwise pretraining parameters and training curves -4. **testTrainingCurves**: Train, and check score vs. iteration -5. **testParamsPostTraining**: validate params match post training -6. **testEvaluation**: test the evaluation performance (post training, if 4 or 5 are true) -7. **testParallelInference**: validate that single net and parallel inference results match -8. **testOverfitting**: sanity check - try to overfit a single example - -See TestCase.java for more details. - - -## Adding a New Integration Test - -The process to add a new test is simple: -1. Add a method that creates and returns a TestCase object (example: testcases/MLPTestCases.getMLPMnist()) -2. Add it as a unit test to IntegrationTests class (example: IntegrationTestsDL4J.testMLPMnist()) -3. Run IntegrationTestBaselineGenerator with the new test case, to generate and save the "known good" results. -4. Run the new integration test to make sure it passes, on both CPU and CUDA backends -5. Commit the generated test resources from step 3 to dl4j-test-resources repo - -Note that IntegrationTestBaselineGenerator assumes you have the dl4j-test-resources cloned parallel to the DL4J mono-repo. \ No newline at end of file diff --git a/deeplearning4j/dl4j-integration-tests/src/test/resources/logback-test.xml b/deeplearning4j/dl4j-integration-tests/src/test/resources/logback-test.xml deleted file mode 100644 index 6be67561e..000000000 --- a/deeplearning4j/dl4j-integration-tests/src/test/resources/logback-test.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - logs/application.log - - %logger{15} - %message%n%xException{5} - - - - - - - %logger{15} - %message%n%xException{5} - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file