[WIP] Ignored TF OP Import test updates (#37)

* new issue for b_d_s (old is closed), new issue for boolean_mask (old is fixed)

Signed-off-by: Ryan Nett <rnett@skymind.io>

* date update

Signed-off-by: Ryan Nett <rnett@skymind.io>

* "embedding_lookup/.*multiple.*" is passing

Signed-off-by: Ryan Nett <rnett@skymind.io>

* new bincount issue (old was fixed)

Signed-off-by: Ryan Nett <rnett@skymind.io>

* date update

Signed-off-by: Ryan Nett <rnett@skymind.io>

* where op comment

Signed-off-by: Ryan Nett <rnett@skymind.io>

* where passing

Signed-off-by: Ryan Nett <rnett@skymind.io>

* ignore gan

Signed-off-by: Ryan Nett <rnett@skymind.io>

* scatter_nd/rank2shape_2indices passing

Signed-off-by: Ryan Nett <rnett@skymind.io>

* topk update, test fix (need to verify)

Signed-off-by: Ryan Nett <rnett@skymind.io>

* batch_to_space issue update

Signed-off-by: Ryan Nett <rnett@skymind.io>

* updates, no change

Signed-off-by: Ryan Nett <rnett@skymind.io>

* date updates

Signed-off-by: Ryan Nett <rnett@skymind.io>

* batch_to_space is fixed

Signed-off-by: Ryan Nett <rnett@skymind.io>

* equality function helper

Signed-off-by: Ryan Nett <rnett@skymind.io>

* topk passing

Signed-off-by: Ryan Nett <rnett@skymind.io>

* dropout equality check

Signed-off-by: Ryan Nett <rnett@skymind.io>

* adding ignores for zoo models

Signed-off-by: Ryan Nett <rnett@skymind.io>

* ignore libnd4j rnn tests

Signed-off-by: Ryan Nett <rnett@skymind.io>

* added batch to space test

Signed-off-by: Ryan Nett <rnett@skymind.io>

* issue comments

Signed-off-by: Ryan Nett <rnett@skymind.io>

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllHelper.java

@@ -16,6 +16,7 @@
 
 package org.nd4j.imports.TFGraphs;
 
+import com.google.common.primitives.Doubles;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.apache.commons.io.FilenameUtils;
@@ -181,7 +182,8 @@ public class TFGraphTestAllHelper {
                     if (tfPred.dataType() != nd4jPred.dataType())
                         nd4jPred = nd4jPred.castTo(tfPred.dataType());
 
-                    boolean eq = tfPred.equals(nd4jPred);
+                    boolean eq = getEqualityFunction(modelName, outputNode, tfPred, nd4jPred).apply(tfPred, nd4jPred);
+
                     if(!eq){
                         //Check for both NaN, both inf
                         if(tfPred.dataType().isFPType() && tfPred.equalShapes(nd4jPred) && tfPred.isNaN().castTo(DataType.INT).sumNumber().intValue() == tfPred.length()
@@ -714,4 +716,50 @@ public class TFGraphTestAllHelper {
         }
         return null;
     }
+
+    public static boolean equalsWithEps(double a, double b){
+        return Math.abs(a - b) <= 0.00001;
+    }
+
+    public static BiFunction<INDArray, INDArray, Boolean> getEqualityFunction(String modelName, String varName, INDArray tf, INDArray sd){
+        if(modelName.startsWith("topk")){
+            return (t, s) -> Nd4j.sort(t, true).equals(Nd4j.sort(s, true));
+        }
+
+        if(modelName.startsWith("alpha_dropout") || modelName.startsWith("layers_dropout"))
+            return (t, s) -> {
+                double[] tfNums = t.ravel().toDoubleVector();
+                double[] sdNums = s.ravel().toDoubleVector();
+
+                Double seen1 = null, seen2 = null;
+                for(int i = 0 ; i < tfNums.length ; i++){
+                    if(!equalsWithEps(tfNums[i], sdNums[i])){
+
+                        // if we have only seen one inequality so far, figure out which is the dropout
+                        if(seen1 != null && seen2 != null){
+                            if(equalsWithEps(tfNums[i], seen1) || equalsWithEps(sdNums[i], seen1)) // the dropout is in seen1
+                                seen2 = null;
+                            else if(equalsWithEps(tfNums[i], seen2) || equalsWithEps(sdNums[i], seen2)){ // the dropout is in seen2
+                                seen1 = seen2;
+                                seen2 = null;
+                            } else // neither match
+                                return false;
+                        }
+
+                        if(seen1 != null){
+                            if(!equalsWithEps(tfNums[i], seen1) && !equalsWithEps(sdNums[i], seen1))
+                                return false;
+                        } else {
+                            seen1 = tfNums[i];
+                            seen2 = sdNums[i];
+                        }
+                    }
+                }
+
+                return true;
+            };
+
+        return Object::equals;
+    }
+
 }

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java

@@ -67,50 +67,29 @@ public class TFGraphTestAllSameDiff {   //Note: Can't extend BaseNd4jTest here a
 
     public static final String[] IGNORE_REGEXES = new String[]{
 
-            //Still failing: 2019/04/08 - https://github.com/deeplearning4j/deeplearning4j/issues/6322 and https://github.com/deeplearning4j/deeplearning4j/issues/6958 issue 1
+            //Still failing: 2019/07/01 - https://github.com/deeplearning4j/deeplearning4j/issues/6322 and https://github.com/eclipse/deeplearning4j/issues/7955
             "broadcast_dynamic_shape/1_4",
             "broadcast_dynamic_shape/2,2_1",
 
-            //Failing 2019/04/09 - JVM Crash - https://github.com/deeplearning4j/deeplearning4j/issues/7495
+            //Failing 2019/07/01 - Libnd4j Concat sizing issue - https://github.com/eclipse/deeplearning4j/issues/7963
             "boolean_mask/.*",
 
-            //Failing 2019/04/08 - Issue 10, https://github.com/deeplearning4j/deeplearning4j/issues/6958
+            //Failing 2019/07/01 - Issue 10, https://github.com/deeplearning4j/deeplearning4j/issues/6958
             "slogdet/.*",
 
-            //Failing 2019/04/08 - Issue 11 - https://github.com/deeplearning4j/deeplearning4j/issues/6958 also exception
+            //Failing 2019/07/01 - https://github.com/eclipse/deeplearning4j/issues/7965
             "bincount/.*",
 
-            //Failures as of 2019/04/08: due to bad gather op - Issue 12 https://github.com/deeplearning4j/deeplearning4j/issues/6958
-            "embedding_lookup/.*multiple.*",
-
-            //Failing 2019/04/08 - Issue 15 https://github.com/deeplearning4j/deeplearning4j/issues/6958
-            "where/cond_only.*",
-
-            //scatter_nd: a few cases failing as of 2019/04/08
-            "scatter_nd/rank2shape_2indices",
-            "scatter_nd/rank3shape_2indices",
-
             //TODO floormod and truncatemod behave differently - i.e., "c" vs. "python" semantics. Need to check implementations too
             "truncatemod/.*",
 
-            //2019/04/08 - This is simply an order issue - need to account for this in test (TF gives no order guarantees)
+            //Still failing as of 2019/07/02 - https://github.com/deeplearning4j/deeplearning4j/issues/6464 - not sure if related to: https://github.com/deeplearning4j/deeplearning4j/issues/6447
-            "topk/.*",
-
-            //Still failing as of 2019/04/08 - https://github.com/deeplearning4j/deeplearning4j/issues/6447
-            "cnn1d_layers/channels_first_b2_k2_s1_d2_SAME",
-            "cnn2d_layers/channels_first_b1_k12_s1_d12_SAME",
-
-            //2019/01/16 - These have a random component so can't be validated using simple .equals... should still be compared, however to check range is sensible etc
-            "alpha_dropout/.*",
-            "layers_dropout/.*",
-
-            //Still failing as of 2019/04/08 - https://github.com/deeplearning4j/deeplearning4j/issues/6464 - not sure if related to: https://github.com/deeplearning4j/deeplearning4j/issues/6447
             "cnn2d_nn/nhwc_b1_k12_s12_d12_SAME",
 
-            //2019/01/08 - No tensorflow op found for SparseTensorDenseAdd
+            //2019/07/02 - No tensorflow op found for SparseTensorDenseAdd
             "confusion/.*",
 
-            //2019/04/08 - Couple of tests failing (InferenceSession issues)
+            //2019/07/02 - Couple of tests failing (InferenceSession issues)
             "rnn/bstack/d_.*",
 
             //2019/05/21 - Failing on AVX2/512 intermittently (Linux, OSX), passing elsewhere

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestList.java

@@ -52,7 +52,7 @@ public class TFGraphTestList {
     public TemporaryFolder testDir = new TemporaryFolder();
 
     public static String[] modelNames = new String[]{
-            "cond/cond_true"
+            "cnn2d_nn/nhwc_b1_k12_s12_d12_SAME"
     };
 
     @After

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestZooModels.java

@@ -57,6 +57,28 @@ public class TFGraphTestZooModels { //Note: Can't extend BaseNd4jTest here as we
     public static TemporaryFolder classTestDir = new TemporaryFolder();
 
     public static final String[] IGNORE_REGEXES = {
+
+
+            //2019/06/28 - Output incorrect, can't debug b/c https://github.com/eclipse/deeplearning4j/issues/7957
+            "cifar10_gan_85",
+
+            //2019/07/03 - java.lang.NullPointerException: varName is marked @NonNull but is null
+            // https://github.com/eclipse/deeplearning4j/issues/7975
+            "alexnet",
+
+            //2019/07/03 - Out of Memory error
+            "compression_residual_gru",
+
+            //2019/07/03 - calculateOutputDataTypes() has not been implemented for org.nd4j.linalg.api.ops.impl.image.ResizeNearestNeighbor
+            // https://github.com/eclipse/deeplearning4j/issues/7976
+            "deeplabv3_xception_ade20k_train",
+
+            //2019/07/03 - o.n.i.g.t.TFGraphMapper - No TensorFlow descriptor found for tensor "sample_sequence/model/h0/attn/MatMul", op "BatchMatMulV2"
+            //org.nd4j.linalg.exception.ND4JIllegalStateException: No tensorflow op found for Multinomial possibly missing operation class?
+            // @ TFGraphMapper.mapNodeType(TFGraphMapper.java:593)
+            // Missing Multinormal op, see https://github.com/eclipse/deeplearning4j/issues/7913
+            "gpt-2_117M",
+
             //2019/05/15 - "Invalid shape for op shape_of: shape has invalid values <= 0: shape=[0]"
             //Also: https://github.com/deeplearning4j/deeplearning4j/issues/7112
             "ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03",
@@ -69,10 +91,19 @@ public class TFGraphTestZooModels { //Note: Can't extend BaseNd4jTest here as we
             "faster_rcnn_resnet101_coco_2018_01_28",
 
             //2019/06/24 - JVM crash on linux-x86_64-cpu-avx2 and -avx512 CI machines only - runs fine elsewhere
-            "deeplabv3_pascal_train_aug_2018_01_04"
+            "deeplabv3_pascal_train_aug_2018_01_04",
     };
 
     public static final String[] IGNORE_REGEXES_LIBND4J_EXEC_ONLY = {
+
+            // Graph wasn't topsorted for all Keras RNNs (possible TF's too)
+            // https://github.com/eclipse/deeplearning4j/issues/7974
+            "PorV-RNN",
+            "temperature_bidirectional_63",
+            "temperature_stacked_63",
+            "text_gen_81",
+
+
             // 2019/05/20 - Buffer is too big to export? https://github.com/deeplearning4j/deeplearning4j/issues/7760
             // File: C:/DL4J/Git/deeplearning4j/libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/flatbuffers.h, Line 668
             //Expression: size() < FLATBUFFERS_MAX_BUFFER_SIZE

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java

@@ -70,6 +70,7 @@ import org.nd4j.linalg.api.ops.impl.transforms.any.IsMax;
 import org.nd4j.linalg.api.ops.impl.transforms.bool.MatchConditionTransform;
 import org.nd4j.linalg.api.ops.impl.transforms.comparison.CompareAndSet;
 import org.nd4j.linalg.api.ops.impl.transforms.comparison.Eps;
+import org.nd4j.linalg.api.ops.impl.transforms.custom.BatchToSpace;
 import org.nd4j.linalg.api.ops.impl.transforms.pairwise.BinaryRelativeError;
 import org.nd4j.linalg.api.ops.impl.transforms.pairwise.Set;
 import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.Axpy;
@@ -7871,6 +7872,22 @@ public class Nd4jTestsC extends BaseNd4jTest {
                         arr[i][j][k][m] = (float) cnt++;
     }
 
+    @Test
+    public void testBatchToSpace(){
+
+        INDArray out = Nd4j.create(DataType.FLOAT, 2, 4, 5);
+        DynamicCustomOp c = new BatchToSpace();
+
+        c.addInputArgument(
+                Nd4j.rand(DataType.FLOAT, new int[]{4, 4, 3}),
+                Nd4j.createFromArray(1, 2),
+                Nd4j.createFromArray(new int[][]{ new int[]{0, 0}, new int[]{0, 1} })
+        );
+        c.addOutputArgument(out);
+        Nd4j.getExecutioner().exec(c);
+
+        assertEquals(Nd4j.createFromArray(1f, 3f, 4f), out);
+    }
 
     private static INDArray fwd(INDArray input, INDArray W, INDArray b){
         INDArray ret = Nd4j.createUninitialized(input.size(0), W.size(1));