Update test suite declarations to ignore tests properly

deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java

@@ -46,6 +46,8 @@ import org.nd4j.linalg.lossfunctions.LossFunctions;
 @Slf4j
 @Tag(TagNames.FILE_IO)
 @NativeTag
+@Tag(TagNames.LONG_TEST)
+@Tag(TagNames.LARGE_RESOURCES)
 public class ParameterServerParallelWrapperTest extends BaseDL4JTest {
 
     @Test

deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java

@@ -38,6 +38,8 @@ import org.nd4j.linalg.factory.Nd4j;
 import static org.junit.jupiter.api.Assertions.*;
 @Tag(TagNames.FILE_IO)
 @NativeTag
+@Tag(TagNames.LARGE_RESOURCES)
+@Tag(TagNames.LONG_TEST)
 public class InplaceParallelInferenceTest extends BaseDL4JTest {
 
     @Test

deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java

@@ -64,6 +64,8 @@ import static org.junit.jupiter.api.Assertions.*;
 @Slf4j
 @Tag(TagNames.FILE_IO)
 @NativeTag
+@Tag(TagNames.LONG_TEST)
+@Tag(TagNames.LARGE_RESOURCES)
 public class ParallelInferenceTest extends BaseDL4JTest {
     private static MultiLayerNetwork model;
     private static DataSetIterator iterator;

nd4j/samediff-import/samediff-import-tensorflow/ops-removed-new.txt

@@ -1,40 +1,7 @@
-input
+in_0
-Reshape/shape
+in_1
-Lenet/conv1/weights
+Add/y
-Lenet/conv1/biases
+in_0/read
-Lenet/conv3/weights
+in_1/read
-Lenet/conv3/biases
+Add
-Lenet/conv5/weights
+Zeta
-Lenet/conv5/biases
-Lenet/fc7/weights
-Lenet/fc7/biases
-Lenet/fc9/weights
-Lenet/fc9/biases
-Lenet/flat6_1/flatten/strided_slice/stack
-Lenet/flat6_1/flatten/strided_slice/stack_1
-Lenet/flat6_1/flatten/strided_slice/stack_2
-Lenet/flat6_1/flatten/Reshape/shape/1
-output/dimension
-Reshape
-Lenet/conv1_1/Conv2D
-Lenet/conv1_1/BiasAdd
-Lenet/conv1_1/Relu
-Lenet/pool2_1/MaxPool
-Lenet/conv3_1/Conv2D
-Lenet/conv3_1/BiasAdd
-Lenet/conv3_1/Relu
-Lenet/pool4_1/MaxPool
-Lenet/conv5_1/Conv2D
-Lenet/conv5_1/BiasAdd
-Lenet/conv5_1/Relu
-Lenet/flat6_1/flatten/Shape
-Lenet/flat6_1/flatten/strided_slice
-Lenet/flat6_1/flatten/Reshape/shape
-Lenet/flat6_1/flatten/Reshape
-Lenet/fc7_1/MatMul
-Lenet/fc7_1/BiasAdd
-Lenet/fc7_1/Relu
-Lenet/fc9_1/MatMul
-Lenet/fc9_1/BiasAdd
-Lenet/fc9_1/Relu
-output

nd4j/samediff-import/samediff-import-tensorflow/pom.xml

@@ -86,6 +86,12 @@
     <profile>
       <id>testresources</id>
     </profile>
+    <profile>
+      <id>nd4j-tests-cpu</id>
+    </profile>
+    <profile>
+      <id>nd4j-tests-cuda</id>
+    </profile>
   </profiles>
 
 </project>

nd4j/samediff-import/samediff-import-tensorflow/src/main/kotlin/org/nd4j/samediff/frameworkimport/tensorflow/definitions/TensorflowOpDeclarations.kt

@@ -826,6 +826,7 @@ val conv3d =  TensorflowMappingProcess(
 
 
 
+
 val divideNoNan = TensorflowMappingProcess(
         opName = "divide_no_nan",
         inputFrameworkOpName = "DivNoNan",

nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java

@@ -277,7 +277,6 @@ public class BERTGraphTest extends BaseNd4jTestWithBackends {
         assertEquals(exp3, softmax.getRow(3));
     }
 
-    @Test //@Disabled   //AB ignored 08/04/2019 until fixed
     @ParameterizedTest
     @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
     public void testBertTraining(Nd4jBackend backend) throws Exception {

nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllHelper.java

@@ -20,6 +20,7 @@
 
 package org.nd4j.imports.tfgraphs;
 
+import org.apache.commons.io.FileUtils;
 import org.nd4j.imports.listeners.ExecPrintListener;
 import org.nd4j.imports.tfgraphs.listener.OpExecOrderListener;
 import lombok.extern.slf4j.Slf4j;
@@ -111,21 +112,6 @@ public class TFGraphTestAllHelper {
 
     public static final DefaultGraphLoader LOADER = new DefaultGraphLoader();
 
-    @BeforeAll
-    public void beforeClass(){
-        log.info("Starting tests for class: " + getClass().getName());
-    }
-
-    @BeforeEach
-    public void setup(){
-        Nd4j.setDataType(DataType.FLOAT);
-    }
-
-    @AfterEach
-    public void tearDown() {
-        NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false);
-        NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false);
-    }
 
     private static ExecutorConfiguration configuration = ExecutorConfiguration.builder()
             .executionMode(ExecutionMode.SEQUENTIAL)
@@ -559,32 +545,29 @@ public class TFGraphTestAllHelper {
         val dtypes = new HashMap<String, DataType>();
 
         List<Pair<Resource,Resource>> resources = new ArrayList<>();
-        if(recursive){
+        if(recursive) {
             String nameRegex = pattern.replace("**.",".*\\.") + "\\.shape";
-//            File baseDir = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString() + "/" + modelName);
-//            baseDir.mkdirs();
-//            baseDir.deleteOnExit();
-//            new ClassPathResource(modelDir).copyDirectory(baseDir);
-
             // checking out, if local folder declared
             String localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar);
-            if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))){
+            if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))) {
                 localPath = FilenameUtils.concat(localPath, "src/main/resources");
             }
 
+
             // baseDir will differ, depending on run mode
             File baseDir = localPath == null ? new File(localTestDir, "extracted/" + modelName) : new File(localPath, base_dir + "/" + modelName);
             String[] arr = baseDir.list();
 
-            if(!baseDir.exists() || arr == null || arr.length == 0){
+            if(!baseDir.exists() || arr == null || arr.length == 0) {
                 // we're skipping extraction if we're using local copy of dl4j-tests-resources
                 if (localPath == null) {
                     baseDir.mkdirs();
-                    baseDir.deleteOnExit();
+                    FileUtils.forceDeleteOnExit(baseDir);
                     String md = modelDir;
                     if(!md.endsWith("/") && !md.endsWith("\\")){
                         md = md + "/";
                     }
+
                     new ClassPathResource(md).copyDirectory(baseDir);
                 } else{
                     throw new IllegalStateException("local directory declared but could not find files: " + baseDir.getAbsolutePath());
@@ -595,7 +578,7 @@ public class TFGraphTestAllHelper {
             LinkedList<File> queue = new LinkedList<>();
             queue.add(baseDir);
 
-            while(!queue.isEmpty()){
+            while(!queue.isEmpty()) {
                 File subdir = queue.remove();
                 File[] files = subdir.listFiles();
                 if (files != null) {
@@ -665,7 +648,7 @@ public class TFGraphTestAllHelper {
 
 //        Preconditions.checkState(!dtypes.isEmpty(), "No datatypes file was found");
 
-        val dtype = Nd4j.dataType();
+
         for (int i = 0; i < resources.size(); i++) {
             URI u = resources.get(i).getFirst().getURI();
             String varName = u.toString();
@@ -730,7 +713,7 @@ public class TFGraphTestAllHelper {
                 }
             } else {
                 int[] varShape = new int[filtered.size()];
-                for( int j=0; j<filtered.size(); j++ ){
+                for( int j = 0; j < filtered.size(); j++) {
                     varShape[j] = Integer.parseInt(filtered.get(j));
                 }
 
@@ -801,8 +784,8 @@ public class TFGraphTestAllHelper {
                                 break;
                             case BOOL:
                                 boolean[] bArr = new boolean[cLines.length];
-                                int z=0;
+                                int z = 0;
-                                while(z < bArr.length){
+                                while(z < bArr.length) {
                                     bArr[z] = parseBoolean(cLines[z]);
                                     z++;
                                 }
@@ -828,7 +811,7 @@ public class TFGraphTestAllHelper {
         return varMap;
     }
 
-    private static long parseLong(String line){
+    private static long parseLong(String line) {
         line = line.trim();       //Handle whitespace
         if(line.matches("-?\\d+\\.0+")){
             //Annoyingly, some integer data is stored with redundant/unnecessary zeros - like "-7.0000000"

nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllLibnd4j.java

@@ -42,6 +42,8 @@ import java.io.IOException;
 import java.util.*;
 import java.util.stream.Stream;
 
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
 
 @Slf4j
 @Tag(TagNames.LONG_TEST)
@@ -130,20 +132,18 @@ public class TFGraphTestAllLibnd4j {   //Note: Can't extend BaseNd4jTest here as
     @ParameterizedTest
     @MethodSource("data")
     public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception {
-
-
         Nd4j.create(1);
         for(String s : TFGraphTestAllSameDiff.IGNORE_REGEXES){
             if(modelName.matches(s)){
                 log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
-                //OpValidationSuite.ignoreFailing();
+                assumeFalse(true);
             }
         }
 
         for(String s : SKIP_FOR_LIBND4J_EXEC) {
             if(modelName.matches(s)){
                 log.info("\n\tIGNORE MODEL ON REGEX - SKIP LIBND4J EXEC ONLY: {} - regex {}", modelName, s);
-                //OpValidationSuite.ignoreFailing();
+                assumeFalse(true);
             }
         }
 

nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java

@@ -24,6 +24,8 @@ import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.junit.jupiter.api.*;
 
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -39,6 +41,8 @@ import java.io.IOException;
 import java.util.*;
 import java.util.stream.Stream;
 
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
 @Slf4j
 @Tag(TagNames.LONG_TEST)
 @Tag(TagNames.LARGE_RESOURCES)
@@ -139,23 +143,7 @@ public class TFGraphTestAllSameDiff {   //Note: Can't extend BaseNd4jTest here a
      */
     private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc");
 
-    @BeforeAll
-    public static void beforeClass() {
-        Nd4j.scalar(1.0);
-        Nd4j.setDataType(DataType.FLOAT);
-        Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC);
-    }
 
-    @BeforeEach
-    public void setup() {
-        Nd4j.setDataType(DataType.FLOAT);
-        Nd4j.getExecutioner().enableDebugMode(true);
-        Nd4j.getExecutioner().enableVerboseMode(true);
-    }
-
-    @AfterEach
-    public void tearDown() {
-    }
 
     public static Stream<Arguments> data() throws IOException {
         val localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar);
@@ -171,19 +159,21 @@ public class TFGraphTestAllSameDiff {   //Note: Can't extend BaseNd4jTest here a
         }
     }
 
-    @ParameterizedTest
+    @ParameterizedTest(name = "{2}")
     @MethodSource("data")
+    //@DisableIfModelFound
     public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception {
         Nd4j.create(1);
         if(EXECUTE_ONLY_MODELS.isEmpty()) {
-            for(String s : IGNORE_REGEXES) {
+            for(String s : IGNORE_REGEXES)  {
                 if(modelName.matches(s)) {
                     log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
-                    //OpValidationSuite.ignoreFailing();
+                    assumeFalse(true);
                 }
             }
         } else if(!EXECUTE_ONLY_MODELS.contains(modelName)) {
             log.info("Not executing " + modelName);
+            assumeFalse(true);
             //OpValidationSuite.ignoreFailing();
         }
 

nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFImportDisableModelsCondition.java

@@ -0,0 +1,155 @@
+/*
+ *
+ *  *  ******************************************************************************
+ *  *  *
+ *  *  *
+ *  *  * This program and the accompanying materials are made available under the
+ *  *  * terms of the Apache License, Version 2.0 which is available at
+ *  *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *  *
+ *  *  *  See the NOTICE file distributed with this work for additional
+ *  *  *  information regarding copyright ownership.
+ *  *  * Unless required by applicable law or agreed to in writing, software
+ *  *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  *  * License for the specific language governing permissions and limitations
+ *  *  * under the License.
+ *  *  *
+ *  *  * SPDX-License-Identifier: Apache-2.0
+ *  *  *****************************************************************************
+ *
+ *
+ */
+
+package org.nd4j.imports.tfgraphs;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
+@Slf4j
+public class TFImportDisableModelsCondition implements ExecutionCondition {
+
+    /**
+     * NOTE: If this is empty or the tests names are wrong,
+     * all tests will trigger an assumeFalse(..) that indicates
+     * the status of the test failing. No tests will run.
+     */
+    public final static List<String> EXECUTE_ONLY_MODELS = Arrays.asList(
+            /*"layers_dropout/rank2_d01_train",
+            "layers_dropout/rank4_d05_train",
+            "layers_dropout/rank3_d05_train_mask2",
+            "layers_dropout/rank4_d05_train_mask",
+            "layers_dropout/rank3_d05_train_mask1",
+            "layers_dropout/rank2_d09_train",
+            "layers_dropout/rank2_d05_train",*/
+
+
+    );
+
+    public static final String[] IGNORE_REGEXES = new String[]{
+            //Failing 2019/09/11 - https://github.com/eclipse/deeplearning4j/issues/7965
+            // Still failing 2020/04/27 java.lang.IllegalStateException: Requested output variable Bincount does not exist in SameDiff instance
+            //Invalid test cases. Verified by running graph against actual TF.
+            "scatter_nd_sub/locking/rank1shape_1indices",
+            "reductions/scatter_update_vector",
+            "reductions/scatter_update_scalar",
+            "emptyArrayTests/scatter_update/rank1_emptyIndices_emptyUpdates",
+            "bincount/rank2_weights",
+            "slogdet/.*",
+            "fused_batch_norm/float16_nhwc",
+            "emptyArrayTests/scatter_update/rank2_emptyIndices_emptyUpdates",
+            //Don't bother to test RNG. We can test subsets of ops with dropout to make sure they are consistent
+            //These tests have random uniform and other RNG in them that don't need to be perfectly compatible to be acceptable.
+            //We need different test cases here.
+            "layers_dropout/.*",
+            //TODO floormod and truncatemod behave differently - i.e., "c" vs. "python" semantics. Need to check implementations too
+            // Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: TruncateMod
+            "truncatemod/.*",
+
+            //2019/09/11 - No tensorflow op found for SparseTensorDenseAdd
+            // 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: SparseTensorDenseAdd
+            "confusion/.*",
+
+            //2019/09/11 - Couple of tests failing (InferenceSession issues)
+            // Still failing 2020/04/27 Requested output variable concat does not exist in SameDiff instance
+
+
+            //2019/05/21 - Failing on windows-x86_64-cuda-9.2 only -
+            "conv_4",
+            "g_09",
+
+            //2019/05/28 - JVM crash on ppc64le only - See issue 7657
+            "g_11",
+
+            //2019/07/09 - Need "Multinomial" op - https://github.com/eclipse/deeplearning4j/issues/7913
+            // Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: Multinomial
+            "multinomial/.*",
+
+            //2019/11/04 AB - disabled, pending libnd4j deconv3d_tf implementation
+            // Still failing 2020/04/27 java.lang.IllegalStateException: Could not find descriptor for op: deconv3d_tf - class: org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv3DTF
+            "conv3d_transpose.*",
+
+            //2019/11/15 - mapping is not present yet https://github.com/eclipse/deepleRaggedRange arning4j/issues/8397
+            // Still failing 2020/04/27 java.lang.AssertionError: Predictions do not match on ragged/reduce_mean/2d_a1, node RaggedReduceMean/truediv
+            "ragged/reduce_mean/.*",
+
+
+            //08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8927
+            "random_gamma/.*",
+
+            //08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8928
+            "Conv3DBackpropInputV2/.*",
+
+
+
+            //12.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8946
+            "non_max_suppression_v4/.*","non_max_suppression_v5/.*",
+
+
+            // 18.05.2020 - :wq:wq
+
+            "random_uniform_int/.*",
+            "random_uniform/.*",
+            "random_poisson_v2/.*"
+    };
+
+    /* As per TFGraphTestList.printArraysDebugging - this field defines a set of regexes for test cases that should have
+       all arrays printed during execution.
+       If a test name matches any regex here, an ExecPrintListener will be added to the listeners, and all output
+       arrays will be printed during execution
+     */
+    private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc");
+
+
+
+    private ExtensionContext.Store getStore(ExtensionContext context) {
+        return context.getStore(ExtensionContext.Namespace.create(getClass(), context.getRequiredTestMethod()));
+    }
+
+    @Override
+    public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) {
+        String modelName = getStore(extensionContext).toString();
+        if(EXECUTE_ONLY_MODELS.isEmpty()) {
+            for(String s : IGNORE_REGEXES)  {
+                if(modelName.matches(s)) {
+                    log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
+                    assumeFalse(true);
+                }
+            }
+        } else if(!EXECUTE_ONLY_MODELS.contains(modelName)) {
+            log.info("Not executing " + modelName);
+            assumeFalse(true);
+            //OpValidationSuite.ignoreFailing();
+        }
+
+
+        return ConditionEvaluationResult.disabled("Method found");
+    }
+}

nd4j/samediff-import/samediff-import-tensorflow/variables-added-new.txt

@@ -1,23 +1,4 @@
-Reshape,Reshape
+in_0/read,in_0/read
-Lenet/conv1_1/Conv2D,Lenet/conv1_1/Conv2D
+in_1/read,in_1/read
-Lenet/conv1_1/BiasAdd,Lenet/conv1_1/BiasAdd
+Add,Add
-Lenet/conv1_1/Relu,Lenet/conv1_1/Relu
+Zeta,Zeta
-Lenet/pool2_1/MaxPool,Lenet/pool2_1/MaxPool
-Lenet/conv3_1/Conv2D,Lenet/conv3_1/Conv2D
-Lenet/conv3_1/BiasAdd,Lenet/conv3_1/BiasAdd
-Lenet/conv3_1/Relu,Lenet/conv3_1/Relu
-Lenet/pool4_1/MaxPool,Lenet/pool4_1/MaxPool
-Lenet/conv5_1/Conv2D,Lenet/conv5_1/Conv2D
-Lenet/conv5_1/BiasAdd,Lenet/conv5_1/BiasAdd
-Lenet/conv5_1/Relu,Lenet/conv5_1/Relu
-Lenet/flat6_1/flatten/Shape,Lenet/flat6_1/flatten/Shape
-Lenet/flat6_1/flatten/strided_slice,Lenet/flat6_1/flatten/strided_slice
-Lenet/flat6_1/flatten/Reshape/shape,Lenet/flat6_1/flatten/Reshape/shape
-Lenet/flat6_1/flatten/Reshape,Lenet/flat6_1/flatten/Reshape
-Lenet/fc7_1/MatMul,Lenet/fc7_1/MatMul
-Lenet/fc7_1/BiasAdd,Lenet/fc7_1/BiasAdd
-Lenet/fc7_1/Relu,Lenet/fc7_1/Relu
-Lenet/fc9_1/MatMul,Lenet/fc9_1/MatMul
-Lenet/fc9_1/BiasAdd,Lenet/fc9_1/BiasAdd
-Lenet/fc9_1/Relu,Lenet/fc9_1/Relu
-output,output