Brutex/cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update test suite declarations to ignore tests properly

Browse Source
This commit is contained in:
agibsonccc 2021-03-27 07:38:03 +09:00
parent d58b87dd7c
commit 763b082f6a
12 changed files with 205 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java
View File

@ -46,6 +46,8 @@ import org.nd4j.linalg.lossfunctions.LossFunctions;
@Slf4j
@Tag(TagNames.FILE_IO)
@NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class ParameterServerParallelWrapperTest extends BaseDL4JTest {
@Test

2
deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java
View File

@ -38,6 +38,8 @@ import org.nd4j.linalg.factory.Nd4j;
import static org.junit.jupiter.api.Assertions.*;
@Tag(TagNames.FILE_IO)
@NativeTag
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public class InplaceParallelInferenceTest extends BaseDL4JTest {
@Test

2
deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java
View File

@ -64,6 +64,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Slf4j
@Tag(TagNames.FILE_IO)
@NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class ParallelInferenceTest extends BaseDL4JTest {
private static MultiLayerNetwork model;
private static DataSetIterator iterator;

47
nd4j/samediff-import/samediff-import-tensorflow/ops-removed-new.txt
View File

@ -1,40 +1,7 @@
input
Reshape/shape
Lenet/conv1/weights
Lenet/conv1/biases
Lenet/conv3/weights
Lenet/conv3/biases
Lenet/conv5/weights
Lenet/conv5/biases
Lenet/fc7/weights
Lenet/fc7/biases
Lenet/fc9/weights
Lenet/fc9/biases
Lenet/flat6_1/flatten/strided_slice/stack
Lenet/flat6_1/flatten/strided_slice/stack_1
Lenet/flat6_1/flatten/strided_slice/stack_2
Lenet/flat6_1/flatten/Reshape/shape/1
output/dimension
Reshape
Lenet/conv1_1/Conv2D
Lenet/conv1_1/BiasAdd
Lenet/conv1_1/Relu
Lenet/pool2_1/MaxPool
Lenet/conv3_1/Conv2D
Lenet/conv3_1/BiasAdd
Lenet/conv3_1/Relu
Lenet/pool4_1/MaxPool
Lenet/conv5_1/Conv2D
Lenet/conv5_1/BiasAdd
Lenet/conv5_1/Relu
Lenet/flat6_1/flatten/Shape
Lenet/flat6_1/flatten/strided_slice
Lenet/flat6_1/flatten/Reshape/shape
Lenet/flat6_1/flatten/Reshape
Lenet/fc7_1/MatMul
Lenet/fc7_1/BiasAdd
Lenet/fc7_1/Relu
Lenet/fc9_1/MatMul
Lenet/fc9_1/BiasAdd
Lenet/fc9_1/Relu
output
in_0
in_1
Add/y
in_0/read
in_1/read
Add
Zeta

6
nd4j/samediff-import/samediff-import-tensorflow/pom.xml
View File

@ -86,6 +86,12 @@
<profile>
<id>testresources</id>
</profile>
<profile>
<id>nd4j-tests-cpu</id>
</profile>
<profile>
<id>nd4j-tests-cuda</id>
</profile>
</profiles>
</project>

1
nd4j/samediff-import/samediff-import-tensorflow/src/main/kotlin/org/nd4j/samediff/frameworkimport/tensorflow/definitions/TensorflowOpDeclarations.kt
View File

@ -826,6 +826,7 @@ val conv3d = TensorflowMappingProcess(
val divideNoNan = TensorflowMappingProcess(
opName = "divide_no_nan",
inputFrameworkOpName = "DivNoNan",

1
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java
View File

@ -277,7 +277,6 @@ public class BERTGraphTest extends BaseNd4jTestWithBackends {
assertEquals(exp3, softmax.getRow(3));
}
@Test //@Disabled //AB ignored 08/04/2019 until fixed
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBertTraining(Nd4jBackend backend) throws Exception {

43
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllHelper.java
View File

@ -20,6 +20,7 @@
package org.nd4j.imports.tfgraphs;
import org.apache.commons.io.FileUtils;
import org.nd4j.imports.listeners.ExecPrintListener;
import org.nd4j.imports.tfgraphs.listener.OpExecOrderListener;
import lombok.extern.slf4j.Slf4j;
@ -111,21 +112,6 @@ public class TFGraphTestAllHelper {
public static final DefaultGraphLoader LOADER = new DefaultGraphLoader();
@BeforeAll
public void beforeClass(){
log.info("Starting tests for class: " + getClass().getName());
}
@BeforeEach
public void setup(){
Nd4j.setDataType(DataType.FLOAT);
}
@AfterEach
public void tearDown() {
NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false);
NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false);
}
private static ExecutorConfiguration configuration = ExecutorConfiguration.builder()
.executionMode(ExecutionMode.SEQUENTIAL)
@ -559,32 +545,29 @@ public class TFGraphTestAllHelper {
val dtypes = new HashMap<String, DataType>();
List<Pair<Resource,Resource>> resources = new ArrayList<>();
if(recursive){
if(recursive) {
String nameRegex = pattern.replace("**.",".*\\.") + "\\.shape";
// File baseDir = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString() + "/" + modelName);
// baseDir.mkdirs();
// baseDir.deleteOnExit();
// new ClassPathResource(modelDir).copyDirectory(baseDir);
// checking out, if local folder declared
String localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar);
if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))){
if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))) {
localPath = FilenameUtils.concat(localPath, "src/main/resources");
}
// baseDir will differ, depending on run mode
File baseDir = localPath == null ? new File(localTestDir, "extracted/" + modelName) : new File(localPath, base_dir + "/" + modelName);
String[] arr = baseDir.list();
if(!baseDir.exists() || arr == null || arr.length == 0){
if(!baseDir.exists() || arr == null || arr.length == 0) {
// we're skipping extraction if we're using local copy of dl4j-tests-resources
if (localPath == null) {
baseDir.mkdirs();
baseDir.deleteOnExit();
FileUtils.forceDeleteOnExit(baseDir);
String md = modelDir;
if(!md.endsWith("/") && !md.endsWith("\\")){
md = md + "/";
}
new ClassPathResource(md).copyDirectory(baseDir);
} else{
throw new IllegalStateException("local directory declared but could not find files: " + baseDir.getAbsolutePath());
@ -595,7 +578,7 @@ public class TFGraphTestAllHelper {
LinkedList<File> queue = new LinkedList<>();
queue.add(baseDir);
while(!queue.isEmpty()){
while(!queue.isEmpty()) {
File subdir = queue.remove();
File[] files = subdir.listFiles();
if (files != null) {
@ -665,7 +648,7 @@ public class TFGraphTestAllHelper {
// Preconditions.checkState(!dtypes.isEmpty(), "No datatypes file was found");
val dtype = Nd4j.dataType();
for (int i = 0; i < resources.size(); i++) {
URI u = resources.get(i).getFirst().getURI();
String varName = u.toString();
@ -730,7 +713,7 @@ public class TFGraphTestAllHelper {
}
} else {
int[] varShape = new int[filtered.size()];
for( int j=0; j<filtered.size(); j++ ){
for( int j = 0; j < filtered.size(); j++) {
varShape[j] = Integer.parseInt(filtered.get(j));
}
@ -801,8 +784,8 @@ public class TFGraphTestAllHelper {
break;
case BOOL:
boolean[] bArr = new boolean[cLines.length];
int z=0;
while(z < bArr.length){
int z = 0;
while(z < bArr.length) {
bArr[z] = parseBoolean(cLines[z]);
z++;
}
@ -828,7 +811,7 @@ public class TFGraphTestAllHelper {
return varMap;
}
private static long parseLong(String line){
private static long parseLong(String line) {
line = line.trim(); //Handle whitespace
if(line.matches("-?\\d+\\.0+")){
//Annoyingly, some integer data is stored with redundant/unnecessary zeros - like "-7.0000000"

8
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllLibnd4j.java
View File

@ -42,6 +42,8 @@ import java.io.IOException;
import java.util.*;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
@Slf4j
@Tag(TagNames.LONG_TEST)
@ -130,20 +132,18 @@ public class TFGraphTestAllLibnd4j { //Note: Can't extend BaseNd4jTest here as
@ParameterizedTest
@MethodSource("data")
public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception {
Nd4j.create(1);
for(String s : TFGraphTestAllSameDiff.IGNORE_REGEXES){
if(modelName.matches(s)){
log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
//OpValidationSuite.ignoreFailing();
assumeFalse(true);
}
}
for(String s : SKIP_FOR_LIBND4J_EXEC) {
if(modelName.matches(s)){
log.info("\n\tIGNORE MODEL ON REGEX - SKIP LIBND4J EXEC ONLY: {} - regex {}", modelName, s);
//OpValidationSuite.ignoreFailing();
assumeFalse(true);
}
}

28
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java
View File

@ -24,6 +24,8 @@ import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@ -39,6 +41,8 @@ import java.io.IOException;
import java.util.*;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
@Slf4j
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
@ -139,23 +143,7 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a
*/
private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc");
@BeforeAll
public static void beforeClass() {
Nd4j.scalar(1.0);
Nd4j.setDataType(DataType.FLOAT);
Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC);
}
@BeforeEach
public void setup() {
Nd4j.setDataType(DataType.FLOAT);
Nd4j.getExecutioner().enableDebugMode(true);
Nd4j.getExecutioner().enableVerboseMode(true);
}
@AfterEach
public void tearDown() {
}
public static Stream<Arguments> data() throws IOException {
val localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar);
@ -171,19 +159,21 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a
}
}
@ParameterizedTest
@ParameterizedTest(name = "{2}")
@MethodSource("data")
//@DisableIfModelFound
public void testOutputOnly(Map<String, INDArray> inputs, Map<String, INDArray> predictions, String modelName, File localTestDir) throws Exception {
Nd4j.create(1);
if(EXECUTE_ONLY_MODELS.isEmpty()) {
for(String s : IGNORE_REGEXES) {
for(String s : IGNORE_REGEXES) {
if(modelName.matches(s)) {
log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
//OpValidationSuite.ignoreFailing();
assumeFalse(true);
}
}
} else if(!EXECUTE_ONLY_MODELS.contains(modelName)) {
log.info("Not executing " + modelName);
assumeFalse(true);
//OpValidationSuite.ignoreFailing();
}

155
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFImportDisableModelsCondition.java Normal file
View File

@ -0,0 +1,155 @@
/*
*
* * ******************************************************************************
* * *
* * *
* * * This program and the accompanying materials are made available under the
* * * terms of the Apache License, Version 2.0 which is available at
* * * https://www.apache.org/licenses/LICENSE-2.0.
* * *
* * * See the NOTICE file distributed with this work for additional
* * * information regarding copyright ownership.
* * * Unless required by applicable law or agreed to in writing, software
* * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * * License for the specific language governing permissions and limitations
* * * under the License.
* * *
* * * SPDX-License-Identifier: Apache-2.0
* * *****************************************************************************
*
*
*/
package org.nd4j.imports.tfgraphs;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
@Slf4j
public class TFImportDisableModelsCondition implements ExecutionCondition {
/**
* NOTE: If this is empty or the tests names are wrong,
* all tests will trigger an assumeFalse(..) that indicates
* the status of the test failing. No tests will run.
*/
public final static List<String> EXECUTE_ONLY_MODELS = Arrays.asList(
/*"layers_dropout/rank2_d01_train",
"layers_dropout/rank4_d05_train",
"layers_dropout/rank3_d05_train_mask2",
"layers_dropout/rank4_d05_train_mask",
"layers_dropout/rank3_d05_train_mask1",
"layers_dropout/rank2_d09_train",
"layers_dropout/rank2_d05_train",*/
);
public static final String[] IGNORE_REGEXES = new String[]{
//Failing 2019/09/11 - https://github.com/eclipse/deeplearning4j/issues/7965
// Still failing 2020/04/27 java.lang.IllegalStateException: Requested output variable Bincount does not exist in SameDiff instance
//Invalid test cases. Verified by running graph against actual TF.
"scatter_nd_sub/locking/rank1shape_1indices",
"reductions/scatter_update_vector",
"reductions/scatter_update_scalar",
"emptyArrayTests/scatter_update/rank1_emptyIndices_emptyUpdates",
"bincount/rank2_weights",
"slogdet/.*",
"fused_batch_norm/float16_nhwc",
"emptyArrayTests/scatter_update/rank2_emptyIndices_emptyUpdates",
//Don't bother to test RNG. We can test subsets of ops with dropout to make sure they are consistent
//These tests have random uniform and other RNG in them that don't need to be perfectly compatible to be acceptable.
//We need different test cases here.
"layers_dropout/.*",
//TODO floormod and truncatemod behave differently - i.e., "c" vs. "python" semantics. Need to check implementations too
// Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: TruncateMod
"truncatemod/.*",
//2019/09/11 - No tensorflow op found for SparseTensorDenseAdd
// 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: SparseTensorDenseAdd
"confusion/.*",
//2019/09/11 - Couple of tests failing (InferenceSession issues)
// Still failing 2020/04/27 Requested output variable concat does not exist in SameDiff instance
//2019/05/21 - Failing on windows-x86_64-cuda-9.2 only -
"conv_4",
"g_09",
//2019/05/28 - JVM crash on ppc64le only - See issue 7657
"g_11",
//2019/07/09 - Need "Multinomial" op - https://github.com/eclipse/deeplearning4j/issues/7913
// Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: Multinomial
"multinomial/.*",
//2019/11/04 AB - disabled, pending libnd4j deconv3d_tf implementation
// Still failing 2020/04/27 java.lang.IllegalStateException: Could not find descriptor for op: deconv3d_tf - class: org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv3DTF
"conv3d_transpose.*",
//2019/11/15 - mapping is not present yet https://github.com/eclipse/deepleRaggedRange arning4j/issues/8397
// Still failing 2020/04/27 java.lang.AssertionError: Predictions do not match on ragged/reduce_mean/2d_a1, node RaggedReduceMean/truediv
"ragged/reduce_mean/.*",
//08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8927
"random_gamma/.*",
//08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8928
"Conv3DBackpropInputV2/.*",
//12.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8946
"non_max_suppression_v4/.*","non_max_suppression_v5/.*",
// 18.05.2020 - :wq:wq
"random_uniform_int/.*",
"random_uniform/.*",
"random_poisson_v2/.*"
};
/* As per TFGraphTestList.printArraysDebugging - this field defines a set of regexes for test cases that should have
all arrays printed during execution.
If a test name matches any regex here, an ExecPrintListener will be added to the listeners, and all output
arrays will be printed during execution
*/
private final List<String> debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc");
private ExtensionContext.Store getStore(ExtensionContext context) {
return context.getStore(ExtensionContext.Namespace.create(getClass(), context.getRequiredTestMethod()));
}
@Override
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) {
String modelName = getStore(extensionContext).toString();
if(EXECUTE_ONLY_MODELS.isEmpty()) {
for(String s : IGNORE_REGEXES) {
if(modelName.matches(s)) {
log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
assumeFalse(true);
}
}
} else if(!EXECUTE_ONLY_MODELS.contains(modelName)) {
log.info("Not executing " + modelName);
assumeFalse(true);
//OpValidationSuite.ignoreFailing();
}
return ConditionEvaluationResult.disabled("Method found");
}
}

27
nd4j/samediff-import/samediff-import-tensorflow/variables-added-new.txt
View File

@ -1,23 +1,4 @@
Reshape,Reshape
Lenet/conv1_1/Conv2D,Lenet/conv1_1/Conv2D
Lenet/conv1_1/BiasAdd,Lenet/conv1_1/BiasAdd
Lenet/conv1_1/Relu,Lenet/conv1_1/Relu
Lenet/pool2_1/MaxPool,Lenet/pool2_1/MaxPool
Lenet/conv3_1/Conv2D,Lenet/conv3_1/Conv2D
Lenet/conv3_1/BiasAdd,Lenet/conv3_1/BiasAdd
Lenet/conv3_1/Relu,Lenet/conv3_1/Relu
Lenet/pool4_1/MaxPool,Lenet/pool4_1/MaxPool
Lenet/conv5_1/Conv2D,Lenet/conv5_1/Conv2D
Lenet/conv5_1/BiasAdd,Lenet/conv5_1/BiasAdd
Lenet/conv5_1/Relu,Lenet/conv5_1/Relu
Lenet/flat6_1/flatten/Shape,Lenet/flat6_1/flatten/Shape
Lenet/flat6_1/flatten/strided_slice,Lenet/flat6_1/flatten/strided_slice
Lenet/flat6_1/flatten/Reshape/shape,Lenet/flat6_1/flatten/Reshape/shape
Lenet/flat6_1/flatten/Reshape,Lenet/flat6_1/flatten/Reshape
Lenet/fc7_1/MatMul,Lenet/fc7_1/MatMul
Lenet/fc7_1/BiasAdd,Lenet/fc7_1/BiasAdd
Lenet/fc7_1/Relu,Lenet/fc7_1/Relu
Lenet/fc9_1/MatMul,Lenet/fc9_1/MatMul
Lenet/fc9_1/BiasAdd,Lenet/fc9_1/BiasAdd
Lenet/fc9_1/Relu,Lenet/fc9_1/Relu
output,output
in_0/read,in_0/read
in_1/read,in_1/read
Add,Add
Zeta,Zeta