Update test suite declarations to ignore tests properly

master
agibsonccc 2021-03-27 07:38:03 +09:00
parent d58b87dd7c
commit 763b082f6a
12 changed files with 205 additions and 117 deletions

View File

@ -46,6 +46,8 @@ import org.nd4j.linalg.lossfunctions.LossFunctions;
@Slf4j @Slf4j
@Tag(TagNames.FILE_IO) @Tag(TagNames.FILE_IO)
@NativeTag @NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class ParameterServerParallelWrapperTest extends BaseDL4JTest { public class ParameterServerParallelWrapperTest extends BaseDL4JTest {
@Test @Test

View File

@ -38,6 +38,8 @@ import org.nd4j.linalg.factory.Nd4j;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
@Tag(TagNames.FILE_IO) @Tag(TagNames.FILE_IO)
@NativeTag @NativeTag
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public class InplaceParallelInferenceTest extends BaseDL4JTest { public class InplaceParallelInferenceTest extends BaseDL4JTest {
@Test @Test

View File

@ -64,6 +64,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Slf4j @Slf4j
@Tag(TagNames.FILE_IO) @Tag(TagNames.FILE_IO)
@NativeTag @NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class ParallelInferenceTest extends BaseDL4JTest { public class ParallelInferenceTest extends BaseDL4JTest {
private static MultiLayerNetwork model; private static MultiLayerNetwork model;
private static DataSetIterator iterator; private static DataSetIterator iterator;

View File

@ -1,40 +1,7 @@
input in_0
Reshape/shape in_1
Lenet/conv1/weights Add/y
Lenet/conv1/biases in_0/read
Lenet/conv3/weights in_1/read
Lenet/conv3/biases Add
Lenet/conv5/weights Zeta
Lenet/conv5/biases
Lenet/fc7/weights
Lenet/fc7/biases
Lenet/fc9/weights
Lenet/fc9/biases
Lenet/flat6_1/flatten/strided_slice/stack
Lenet/flat6_1/flatten/strided_slice/stack_1
Lenet/flat6_1/flatten/strided_slice/stack_2
Lenet/flat6_1/flatten/Reshape/shape/1
output/dimension
Reshape
Lenet/conv1_1/Conv2D
Lenet/conv1_1/BiasAdd
Lenet/conv1_1/Relu
Lenet/pool2_1/MaxPool
Lenet/conv3_1/Conv2D
Lenet/conv3_1/BiasAdd
Lenet/conv3_1/Relu
Lenet/pool4_1/MaxPool
Lenet/conv5_1/Conv2D
Lenet/conv5_1/BiasAdd
Lenet/conv5_1/Relu
Lenet/flat6_1/flatten/Shape
Lenet/flat6_1/flatten/strided_slice
Lenet/flat6_1/flatten/Reshape/shape
Lenet/flat6_1/flatten/Reshape
Lenet/fc7_1/MatMul
Lenet/fc7_1/BiasAdd
Lenet/fc7_1/Relu
Lenet/fc9_1/MatMul
Lenet/fc9_1/BiasAdd
Lenet/fc9_1/Relu
output

View File

@ -86,6 +86,12 @@
<profile> <profile>
<id>testresources</id> <id>testresources</id>
</profile> </profile>
<profile>
<id>nd4j-tests-cpu</id>
</profile>
<profile>
<id>nd4j-tests-cuda</id>
</profile>
</profiles> </profiles>
</project> </project>

View File

@ -826,6 +826,7 @@ val conv3d = TensorflowMappingProcess(
val divideNoNan = TensorflowMappingProcess( val divideNoNan = TensorflowMappingProcess(
opName = "divide_no_nan", opName = "divide_no_nan",
inputFrameworkOpName = "DivNoNan", inputFrameworkOpName = "DivNoNan",

View File

@ -277,7 +277,6 @@ public class BERTGraphTest extends BaseNd4jTestWithBackends {
assertEquals(exp3, softmax.getRow(3)); assertEquals(exp3, softmax.getRow(3));
} }
@Test //@Disabled //AB ignored 08/04/2019 until fixed
@ParameterizedTest @ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBertTraining(Nd4jBackend backend) throws Exception { public void testBertTraining(Nd4jBackend backend) throws Exception {

View File

@ -20,6 +20,7 @@
package org.nd4j.imports.tfgraphs; package org.nd4j.imports.tfgraphs;
import org.apache.commons.io.FileUtils;
import org.nd4j.imports.listeners.ExecPrintListener; import org.nd4j.imports.listeners.ExecPrintListener;
import org.nd4j.imports.tfgraphs.listener.OpExecOrderListener; import org.nd4j.imports.tfgraphs.listener.OpExecOrderListener;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -111,21 +112,6 @@ public class TFGraphTestAllHelper {
public static final DefaultGraphLoader LOADER = new DefaultGraphLoader(); public static final DefaultGraphLoader LOADER = new DefaultGraphLoader();
@BeforeAll
public void beforeClass(){
log.info("Starting tests for class: " + getClass().getName());
}
@BeforeEach
public void setup(){
Nd4j.setDataType(DataType.FLOAT);
}
@AfterEach
public void tearDown() {
NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false);
NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false);
}
private static ExecutorConfiguration configuration = ExecutorConfiguration.builder() private static ExecutorConfiguration configuration = ExecutorConfiguration.builder()
.executionMode(ExecutionMode.SEQUENTIAL) .executionMode(ExecutionMode.SEQUENTIAL)
@ -561,17 +547,13 @@ public class TFGraphTestAllHelper {
List<Pair<Resource,Resource>> resources = new ArrayList<>(); List<Pair<Resource,Resource>> resources = new ArrayList<>();
if(recursive) { if(recursive) {
String nameRegex = pattern.replace("**.",".*\\.") + "\\.shape"; String nameRegex = pattern.replace("**.",".*\\.") + "\\.shape";
// File baseDir = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString() + "/" + modelName);
// baseDir.mkdirs();
// baseDir.deleteOnExit();
// new ClassPathResource(modelDir).copyDirectory(baseDir);
// checking out, if local folder declared // checking out, if local folder declared
String localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar); String localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar);
if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))) { if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))) {
localPath = FilenameUtils.concat(localPath, "src/main/resources"); localPath = FilenameUtils.concat(localPath, "src/main/resources");
} }
// baseDir will differ, depending on run mode // baseDir will differ, depending on run mode
File baseDir = localPath == null ? new File(localTestDir, "extracted/" + modelName) : new File(localPath, base_dir + "/" + modelName); File baseDir = localPath == null ? new File(localTestDir, "extracted/" + modelName) : new File(localPath, base_dir + "/" + modelName);
String[] arr = baseDir.list(); String[] arr = baseDir.list();
@ -580,11 +562,12 @@ public class TFGraphTestAllHelper {
// we're skipping extraction if we're using local copy of dl4j-tests-resources // we're skipping extraction if we're using local copy of dl4j-tests-resources
if (localPath == null) { if (localPath == null) {
baseDir.mkdirs(); baseDir.mkdirs();
baseDir.deleteOnExit(); FileUtils.forceDeleteOnExit(baseDir);
String md = modelDir; String md = modelDir;
if(!md.endsWith("/") && !md.endsWith("\\")){ if(!md.endsWith("/") && !md.endsWith("\\")){
md = md + "/"; md = md + "/";
} }
new ClassPathResource(md).copyDirectory(baseDir); new ClassPathResource(md).copyDirectory(baseDir);
} else{ } else{
throw new IllegalStateException("local directory declared but could not find files: " + baseDir.getAbsolutePath()); throw new IllegalStateException("local directory declared but could not find files: " + baseDir.getAbsolutePath());
@ -665,7 +648,7 @@ public class TFGraphTestAllHelper {
// Preconditions.checkState(!dtypes.isEmpty(), "No datatypes file was found"); // Preconditions.checkState(!dtypes.isEmpty(), "No datatypes file was found");
val dtype = Nd4j.dataType();
for (int i = 0; i < resources.size(); i++) { for (int i = 0; i < resources.size(); i++) {
URI u = resources.get(i).getFirst().getURI(); URI u = resources.get(i).getFirst().getURI();
String varName = u.toString(); String varName = u.toString();

View File

@ -42,6 +42,8 @@ import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
@Slf4j @Slf4j
@Tag(TagNames.LONG_TEST) @Tag(TagNames.LONG_TEST)
@ -130,20 +132,18 @@ public class TFGraphTestAllLibnd4j { //Note: Can't extend BaseNd4jTest here as
@ParameterizedTest @ParameterizedTest
@MethodSource("data") @MethodSource("data")
public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception { public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception {
Nd4j.create(1); Nd4j.create(1);
for(String s : TFGraphTestAllSameDiff.IGNORE_REGEXES){ for(String s : TFGraphTestAllSameDiff.IGNORE_REGEXES){
if(modelName.matches(s)){ if(modelName.matches(s)){
log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s); log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
//OpValidationSuite.ignoreFailing(); assumeFalse(true);
} }
} }
for(String s : SKIP_FOR_LIBND4J_EXEC) { for(String s : SKIP_FOR_LIBND4J_EXEC) {
if(modelName.matches(s)){ if(modelName.matches(s)){
log.info("\n\tIGNORE MODEL ON REGEX - SKIP LIBND4J EXEC ONLY: {} - regex {}", modelName, s); log.info("\n\tIGNORE MODEL ON REGEX - SKIP LIBND4J EXEC ONLY: {} - regex {}", modelName, s);
//OpValidationSuite.ignoreFailing(); assumeFalse(true);
} }
} }

View File

@ -24,6 +24,8 @@ import lombok.extern.slf4j.Slf4j;
import lombok.val; import lombok.val;
import org.junit.jupiter.api.*; import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.MethodSource;
@ -39,6 +41,8 @@ import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
@Slf4j @Slf4j
@Tag(TagNames.LONG_TEST) @Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES) @Tag(TagNames.LARGE_RESOURCES)
@ -139,23 +143,7 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a
*/ */
private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc"); private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc");
@BeforeAll
public static void beforeClass() {
Nd4j.scalar(1.0);
Nd4j.setDataType(DataType.FLOAT);
Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC);
}
@BeforeEach
public void setup() {
Nd4j.setDataType(DataType.FLOAT);
Nd4j.getExecutioner().enableDebugMode(true);
Nd4j.getExecutioner().enableVerboseMode(true);
}
@AfterEach
public void tearDown() {
}
public static Stream<Arguments> data() throws IOException { public static Stream<Arguments> data() throws IOException {
val localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar); val localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar);
@ -171,19 +159,21 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a
} }
} }
@ParameterizedTest @ParameterizedTest(name = "{2}")
@MethodSource("data") @MethodSource("data")
//@DisableIfModelFound
public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception { public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception {
Nd4j.create(1); Nd4j.create(1);
if(EXECUTE_ONLY_MODELS.isEmpty()) { if(EXECUTE_ONLY_MODELS.isEmpty()) {
for(String s : IGNORE_REGEXES) { for(String s : IGNORE_REGEXES) {
if(modelName.matches(s)) { if(modelName.matches(s)) {
log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s); log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
//OpValidationSuite.ignoreFailing(); assumeFalse(true);
} }
} }
} else if(!EXECUTE_ONLY_MODELS.contains(modelName)) { } else if(!EXECUTE_ONLY_MODELS.contains(modelName)) {
log.info("Not executing " + modelName); log.info("Not executing " + modelName);
assumeFalse(true);
//OpValidationSuite.ignoreFailing(); //OpValidationSuite.ignoreFailing();
} }

View File

@ -0,0 +1,155 @@
/*
*
* * ******************************************************************************
* * *
* * *
* * * This program and the accompanying materials are made available under the
* * * terms of the Apache License, Version 2.0 which is available at
* * * https://www.apache.org/licenses/LICENSE-2.0.
* * *
* * * See the NOTICE file distributed with this work for additional
* * * information regarding copyright ownership.
* * * Unless required by applicable law or agreed to in writing, software
* * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * * License for the specific language governing permissions and limitations
* * * under the License.
* * *
* * * SPDX-License-Identifier: Apache-2.0
* * *****************************************************************************
*
*
*/
package org.nd4j.imports.tfgraphs;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
@Slf4j
public class TFImportDisableModelsCondition implements ExecutionCondition {
/**
* NOTE: If this is empty or the tests names are wrong,
* all tests will trigger an assumeFalse(..) that indicates
* the status of the test failing. No tests will run.
*/
public final static List<String> EXECUTE_ONLY_MODELS = Arrays.asList(
/*"layers_dropout/rank2_d01_train",
"layers_dropout/rank4_d05_train",
"layers_dropout/rank3_d05_train_mask2",
"layers_dropout/rank4_d05_train_mask",
"layers_dropout/rank3_d05_train_mask1",
"layers_dropout/rank2_d09_train",
"layers_dropout/rank2_d05_train",*/
);
public static final String[] IGNORE_REGEXES = new String[]{
//Failing 2019/09/11 - https://github.com/eclipse/deeplearning4j/issues/7965
// Still failing 2020/04/27 java.lang.IllegalStateException: Requested output variable Bincount does not exist in SameDiff instance
//Invalid test cases. Verified by running graph against actual TF.
"scatter_nd_sub/locking/rank1shape_1indices",
"reductions/scatter_update_vector",
"reductions/scatter_update_scalar",
"emptyArrayTests/scatter_update/rank1_emptyIndices_emptyUpdates",
"bincount/rank2_weights",
"slogdet/.*",
"fused_batch_norm/float16_nhwc",
"emptyArrayTests/scatter_update/rank2_emptyIndices_emptyUpdates",
//Don't bother to test RNG. We can test subsets of ops with dropout to make sure they are consistent
//These tests have random uniform and other RNG in them that don't need to be perfectly compatible to be acceptable.
//We need different test cases here.
"layers_dropout/.*",
//TODO floormod and truncatemod behave differently - i.e., "c" vs. "python" semantics. Need to check implementations too
// Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: TruncateMod
"truncatemod/.*",
//2019/09/11 - No tensorflow op found for SparseTensorDenseAdd
// 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: SparseTensorDenseAdd
"confusion/.*",
//2019/09/11 - Couple of tests failing (InferenceSession issues)
// Still failing 2020/04/27 Requested output variable concat does not exist in SameDiff instance
//2019/05/21 - Failing on windows-x86_64-cuda-9.2 only -
"conv_4",
"g_09",
//2019/05/28 - JVM crash on ppc64le only - See issue 7657
"g_11",
//2019/07/09 - Need "Multinomial" op - https://github.com/eclipse/deeplearning4j/issues/7913
// Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: Multinomial
"multinomial/.*",
//2019/11/04 AB - disabled, pending libnd4j deconv3d_tf implementation
// Still failing 2020/04/27 java.lang.IllegalStateException: Could not find descriptor for op: deconv3d_tf - class: org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv3DTF
"conv3d_transpose.*",
//2019/11/15 - mapping is not present yet https://github.com/eclipse/deepleRaggedRange arning4j/issues/8397
// Still failing 2020/04/27 java.lang.AssertionError: Predictions do not match on ragged/reduce_mean/2d_a1, node RaggedReduceMean/truediv
"ragged/reduce_mean/.*",
//08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8927
"random_gamma/.*",
//08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8928
"Conv3DBackpropInputV2/.*",
//12.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8946
"non_max_suppression_v4/.*","non_max_suppression_v5/.*",
// 18.05.2020 - :wq:wq
"random_uniform_int/.*",
"random_uniform/.*",
"random_poisson_v2/.*"
};
/* As per TFGraphTestList.printArraysDebugging - this field defines a set of regexes for test cases that should have
all arrays printed during execution.
If a test name matches any regex here, an ExecPrintListener will be added to the listeners, and all output
arrays will be printed during execution
*/
private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc");
private ExtensionContext.Store getStore(ExtensionContext context) {
return context.getStore(ExtensionContext.Namespace.create(getClass(), context.getRequiredTestMethod()));
}
@Override
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) {
String modelName = getStore(extensionContext).toString();
if(EXECUTE_ONLY_MODELS.isEmpty()) {
for(String s : IGNORE_REGEXES) {
if(modelName.matches(s)) {
log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
assumeFalse(true);
}
}
} else if(!EXECUTE_ONLY_MODELS.contains(modelName)) {
log.info("Not executing " + modelName);
assumeFalse(true);
//OpValidationSuite.ignoreFailing();
}
return ConditionEvaluationResult.disabled("Method found");
}
}

View File

@ -1,23 +1,4 @@
Reshape,Reshape in_0/read,in_0/read
Lenet/conv1_1/Conv2D,Lenet/conv1_1/Conv2D in_1/read,in_1/read
Lenet/conv1_1/BiasAdd,Lenet/conv1_1/BiasAdd Add,Add
Lenet/conv1_1/Relu,Lenet/conv1_1/Relu Zeta,Zeta
Lenet/pool2_1/MaxPool,Lenet/pool2_1/MaxPool
Lenet/conv3_1/Conv2D,Lenet/conv3_1/Conv2D
Lenet/conv3_1/BiasAdd,Lenet/conv3_1/BiasAdd
Lenet/conv3_1/Relu,Lenet/conv3_1/Relu
Lenet/pool4_1/MaxPool,Lenet/pool4_1/MaxPool
Lenet/conv5_1/Conv2D,Lenet/conv5_1/Conv2D
Lenet/conv5_1/BiasAdd,Lenet/conv5_1/BiasAdd
Lenet/conv5_1/Relu,Lenet/conv5_1/Relu
Lenet/flat6_1/flatten/Shape,Lenet/flat6_1/flatten/Shape
Lenet/flat6_1/flatten/strided_slice,Lenet/flat6_1/flatten/strided_slice
Lenet/flat6_1/flatten/Reshape/shape,Lenet/flat6_1/flatten/Reshape/shape
Lenet/flat6_1/flatten/Reshape,Lenet/flat6_1/flatten/Reshape
Lenet/fc7_1/MatMul,Lenet/fc7_1/MatMul
Lenet/fc7_1/BiasAdd,Lenet/fc7_1/BiasAdd
Lenet/fc7_1/Relu,Lenet/fc7_1/Relu
Lenet/fc9_1/MatMul,Lenet/fc9_1/MatMul
Lenet/fc9_1/BiasAdd,Lenet/fc9_1/BiasAdd
Lenet/fc9_1/Relu,Lenet/fc9_1/Relu
output,output