diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java index 8700d121c..bee9bb710 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java @@ -46,6 +46,8 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; @Slf4j @Tag(TagNames.FILE_IO) @NativeTag +@Tag(TagNames.LONG_TEST) +@Tag(TagNames.LARGE_RESOURCES) public class ParameterServerParallelWrapperTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java index 35940a519..7e35c4cd2 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java @@ -38,6 +38,8 @@ import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.*; @Tag(TagNames.FILE_IO) @NativeTag +@Tag(TagNames.LARGE_RESOURCES) +@Tag(TagNames.LONG_TEST) public class InplaceParallelInferenceTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java index 8c25bae54..f5a3eaaeb 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java @@ -64,6 +64,8 @@ import static org.junit.jupiter.api.Assertions.*; @Slf4j @Tag(TagNames.FILE_IO) @NativeTag +@Tag(TagNames.LONG_TEST) +@Tag(TagNames.LARGE_RESOURCES) public class ParallelInferenceTest extends BaseDL4JTest { private static MultiLayerNetwork model; private static DataSetIterator iterator; diff --git a/nd4j/samediff-import/samediff-import-tensorflow/ops-removed-new.txt b/nd4j/samediff-import/samediff-import-tensorflow/ops-removed-new.txt index ed18a8292..49a83c42f 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/ops-removed-new.txt +++ b/nd4j/samediff-import/samediff-import-tensorflow/ops-removed-new.txt @@ -1,40 +1,7 @@ -input -Reshape/shape -Lenet/conv1/weights -Lenet/conv1/biases -Lenet/conv3/weights -Lenet/conv3/biases -Lenet/conv5/weights -Lenet/conv5/biases -Lenet/fc7/weights -Lenet/fc7/biases -Lenet/fc9/weights -Lenet/fc9/biases -Lenet/flat6_1/flatten/strided_slice/stack -Lenet/flat6_1/flatten/strided_slice/stack_1 -Lenet/flat6_1/flatten/strided_slice/stack_2 -Lenet/flat6_1/flatten/Reshape/shape/1 -output/dimension -Reshape -Lenet/conv1_1/Conv2D -Lenet/conv1_1/BiasAdd -Lenet/conv1_1/Relu -Lenet/pool2_1/MaxPool -Lenet/conv3_1/Conv2D -Lenet/conv3_1/BiasAdd -Lenet/conv3_1/Relu -Lenet/pool4_1/MaxPool -Lenet/conv5_1/Conv2D -Lenet/conv5_1/BiasAdd -Lenet/conv5_1/Relu -Lenet/flat6_1/flatten/Shape -Lenet/flat6_1/flatten/strided_slice -Lenet/flat6_1/flatten/Reshape/shape -Lenet/flat6_1/flatten/Reshape -Lenet/fc7_1/MatMul -Lenet/fc7_1/BiasAdd -Lenet/fc7_1/Relu -Lenet/fc9_1/MatMul -Lenet/fc9_1/BiasAdd -Lenet/fc9_1/Relu -output +in_0 +in_1 +Add/y +in_0/read +in_1/read +Add +Zeta diff --git a/nd4j/samediff-import/samediff-import-tensorflow/pom.xml b/nd4j/samediff-import/samediff-import-tensorflow/pom.xml index dc4a5f5b6..081f1eb9d 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/pom.xml +++ b/nd4j/samediff-import/samediff-import-tensorflow/pom.xml @@ -86,6 +86,12 @@ testresources + + nd4j-tests-cpu + + + nd4j-tests-cuda + diff --git a/nd4j/samediff-import/samediff-import-tensorflow/src/main/kotlin/org/nd4j/samediff/frameworkimport/tensorflow/definitions/TensorflowOpDeclarations.kt b/nd4j/samediff-import/samediff-import-tensorflow/src/main/kotlin/org/nd4j/samediff/frameworkimport/tensorflow/definitions/TensorflowOpDeclarations.kt index 00fdf907d..6ee60078d 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/src/main/kotlin/org/nd4j/samediff/frameworkimport/tensorflow/definitions/TensorflowOpDeclarations.kt +++ b/nd4j/samediff-import/samediff-import-tensorflow/src/main/kotlin/org/nd4j/samediff/frameworkimport/tensorflow/definitions/TensorflowOpDeclarations.kt @@ -826,6 +826,7 @@ val conv3d = TensorflowMappingProcess( + val divideNoNan = TensorflowMappingProcess( opName = "divide_no_nan", inputFrameworkOpName = "DivNoNan", diff --git a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java index e4e548c11..2845a231a 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java +++ b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java @@ -277,7 +277,6 @@ public class BERTGraphTest extends BaseNd4jTestWithBackends { assertEquals(exp3, softmax.getRow(3)); } - @Test //@Disabled //AB ignored 08/04/2019 until fixed @ParameterizedTest @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") public void testBertTraining(Nd4jBackend backend) throws Exception { diff --git a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllHelper.java b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllHelper.java index 6a069d545..2bb6aa5ab 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllHelper.java +++ b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllHelper.java @@ -20,6 +20,7 @@ package org.nd4j.imports.tfgraphs; +import org.apache.commons.io.FileUtils; import org.nd4j.imports.listeners.ExecPrintListener; import org.nd4j.imports.tfgraphs.listener.OpExecOrderListener; import lombok.extern.slf4j.Slf4j; @@ -111,21 +112,6 @@ public class TFGraphTestAllHelper { public static final DefaultGraphLoader LOADER = new DefaultGraphLoader(); - @BeforeAll - public void beforeClass(){ - log.info("Starting tests for class: " + getClass().getName()); - } - - @BeforeEach - public void setup(){ - Nd4j.setDataType(DataType.FLOAT); - } - - @AfterEach - public void tearDown() { - NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false); - NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false); - } private static ExecutorConfiguration configuration = ExecutorConfiguration.builder() .executionMode(ExecutionMode.SEQUENTIAL) @@ -559,32 +545,29 @@ public class TFGraphTestAllHelper { val dtypes = new HashMap(); List> resources = new ArrayList<>(); - if(recursive){ + if(recursive) { String nameRegex = pattern.replace("**.",".*\\.") + "\\.shape"; -// File baseDir = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString() + "/" + modelName); -// baseDir.mkdirs(); -// baseDir.deleteOnExit(); -// new ClassPathResource(modelDir).copyDirectory(baseDir); - // checking out, if local folder declared String localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar); - if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))){ + if(localPath != null && (!localPath.contains("src/main/resources") && !localPath.contains("src\\main\\resources"))) { localPath = FilenameUtils.concat(localPath, "src/main/resources"); } + // baseDir will differ, depending on run mode File baseDir = localPath == null ? new File(localTestDir, "extracted/" + modelName) : new File(localPath, base_dir + "/" + modelName); String[] arr = baseDir.list(); - if(!baseDir.exists() || arr == null || arr.length == 0){ + if(!baseDir.exists() || arr == null || arr.length == 0) { // we're skipping extraction if we're using local copy of dl4j-tests-resources if (localPath == null) { baseDir.mkdirs(); - baseDir.deleteOnExit(); + FileUtils.forceDeleteOnExit(baseDir); String md = modelDir; if(!md.endsWith("/") && !md.endsWith("\\")){ md = md + "/"; } + new ClassPathResource(md).copyDirectory(baseDir); } else{ throw new IllegalStateException("local directory declared but could not find files: " + baseDir.getAbsolutePath()); @@ -595,7 +578,7 @@ public class TFGraphTestAllHelper { LinkedList queue = new LinkedList<>(); queue.add(baseDir); - while(!queue.isEmpty()){ + while(!queue.isEmpty()) { File subdir = queue.remove(); File[] files = subdir.listFiles(); if (files != null) { @@ -665,7 +648,7 @@ public class TFGraphTestAllHelper { // Preconditions.checkState(!dtypes.isEmpty(), "No datatypes file was found"); - val dtype = Nd4j.dataType(); + for (int i = 0; i < resources.size(); i++) { URI u = resources.get(i).getFirst().getURI(); String varName = u.toString(); @@ -730,7 +713,7 @@ public class TFGraphTestAllHelper { } } else { int[] varShape = new int[filtered.size()]; - for( int j=0; j inputs, Map predictions, String modelName, File localTestDir) throws Exception { - - Nd4j.create(1); for(String s : TFGraphTestAllSameDiff.IGNORE_REGEXES){ if(modelName.matches(s)){ log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s); - //OpValidationSuite.ignoreFailing(); + assumeFalse(true); } } for(String s : SKIP_FOR_LIBND4J_EXEC) { if(modelName.matches(s)){ log.info("\n\tIGNORE MODEL ON REGEX - SKIP LIBND4J EXEC ONLY: {} - regex {}", modelName, s); - //OpValidationSuite.ignoreFailing(); + assumeFalse(true); } } diff --git a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java index 84bc7531a..9c55d9b12 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java +++ b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java @@ -24,6 +24,8 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.jupiter.api.*; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -39,6 +41,8 @@ import java.io.IOException; import java.util.*; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + @Slf4j @Tag(TagNames.LONG_TEST) @Tag(TagNames.LARGE_RESOURCES) @@ -139,23 +143,7 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a */ private final List debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc"); - @BeforeAll - public static void beforeClass() { - Nd4j.scalar(1.0); - Nd4j.setDataType(DataType.FLOAT); - Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); - } - @BeforeEach - public void setup() { - Nd4j.setDataType(DataType.FLOAT); - Nd4j.getExecutioner().enableDebugMode(true); - Nd4j.getExecutioner().enableVerboseMode(true); - } - - @AfterEach - public void tearDown() { - } public static Stream data() throws IOException { val localPath = System.getenv(TFGraphTestAllHelper.resourceFolderVar); @@ -171,19 +159,21 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a } } - @ParameterizedTest + @ParameterizedTest(name = "{2}") @MethodSource("data") + //@DisableIfModelFound public void testOutputOnly(Map inputs, Map predictions, String modelName, File localTestDir) throws Exception { Nd4j.create(1); if(EXECUTE_ONLY_MODELS.isEmpty()) { - for(String s : IGNORE_REGEXES) { + for(String s : IGNORE_REGEXES) { if(modelName.matches(s)) { log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s); - //OpValidationSuite.ignoreFailing(); + assumeFalse(true); } } } else if(!EXECUTE_ONLY_MODELS.contains(modelName)) { log.info("Not executing " + modelName); + assumeFalse(true); //OpValidationSuite.ignoreFailing(); } diff --git a/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFImportDisableModelsCondition.java b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFImportDisableModelsCondition.java new file mode 100644 index 000000000..e7d8b3fa4 --- /dev/null +++ b/nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFImportDisableModelsCondition.java @@ -0,0 +1,155 @@ +/* + * + * * ****************************************************************************** + * * * + * * * + * * * This program and the accompanying materials are made available under the + * * * terms of the Apache License, Version 2.0 which is available at + * * * https://www.apache.org/licenses/LICENSE-2.0. + * * * + * * * See the NOTICE file distributed with this work for additional + * * * information regarding copyright ownership. + * * * Unless required by applicable law or agreed to in writing, software + * * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * * License for the specific language governing permissions and limitations + * * * under the License. + * * * + * * * SPDX-License-Identifier: Apache-2.0 + * * ***************************************************************************** + * + * + */ + +package org.nd4j.imports.tfgraphs; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +@Slf4j +public class TFImportDisableModelsCondition implements ExecutionCondition { + + /** + * NOTE: If this is empty or the tests names are wrong, + * all tests will trigger an assumeFalse(..) that indicates + * the status of the test failing. No tests will run. + */ + public final static List EXECUTE_ONLY_MODELS = Arrays.asList( + /*"layers_dropout/rank2_d01_train", + "layers_dropout/rank4_d05_train", + "layers_dropout/rank3_d05_train_mask2", + "layers_dropout/rank4_d05_train_mask", + "layers_dropout/rank3_d05_train_mask1", + "layers_dropout/rank2_d09_train", + "layers_dropout/rank2_d05_train",*/ + + + ); + + public static final String[] IGNORE_REGEXES = new String[]{ + //Failing 2019/09/11 - https://github.com/eclipse/deeplearning4j/issues/7965 + // Still failing 2020/04/27 java.lang.IllegalStateException: Requested output variable Bincount does not exist in SameDiff instance + //Invalid test cases. Verified by running graph against actual TF. + "scatter_nd_sub/locking/rank1shape_1indices", + "reductions/scatter_update_vector", + "reductions/scatter_update_scalar", + "emptyArrayTests/scatter_update/rank1_emptyIndices_emptyUpdates", + "bincount/rank2_weights", + "slogdet/.*", + "fused_batch_norm/float16_nhwc", + "emptyArrayTests/scatter_update/rank2_emptyIndices_emptyUpdates", + //Don't bother to test RNG. We can test subsets of ops with dropout to make sure they are consistent + //These tests have random uniform and other RNG in them that don't need to be perfectly compatible to be acceptable. + //We need different test cases here. + "layers_dropout/.*", + //TODO floormod and truncatemod behave differently - i.e., "c" vs. "python" semantics. Need to check implementations too + // Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: TruncateMod + "truncatemod/.*", + + //2019/09/11 - No tensorflow op found for SparseTensorDenseAdd + // 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: SparseTensorDenseAdd + "confusion/.*", + + //2019/09/11 - Couple of tests failing (InferenceSession issues) + // Still failing 2020/04/27 Requested output variable concat does not exist in SameDiff instance + + + //2019/05/21 - Failing on windows-x86_64-cuda-9.2 only - + "conv_4", + "g_09", + + //2019/05/28 - JVM crash on ppc64le only - See issue 7657 + "g_11", + + //2019/07/09 - Need "Multinomial" op - https://github.com/eclipse/deeplearning4j/issues/7913 + // Still failing 2020/04/27 java.lang.IllegalStateException: Could not find class for TF Ops: Multinomial + "multinomial/.*", + + //2019/11/04 AB - disabled, pending libnd4j deconv3d_tf implementation + // Still failing 2020/04/27 java.lang.IllegalStateException: Could not find descriptor for op: deconv3d_tf - class: org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv3DTF + "conv3d_transpose.*", + + //2019/11/15 - mapping is not present yet https://github.com/eclipse/deepleRaggedRange arning4j/issues/8397 + // Still failing 2020/04/27 java.lang.AssertionError: Predictions do not match on ragged/reduce_mean/2d_a1, node RaggedReduceMean/truediv + "ragged/reduce_mean/.*", + + + //08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8927 + "random_gamma/.*", + + //08.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8928 + "Conv3DBackpropInputV2/.*", + + + + //12.05.2020 - https://github.com/eclipse/deeplearning4j/issues/8946 + "non_max_suppression_v4/.*","non_max_suppression_v5/.*", + + + // 18.05.2020 - :wq:wq + + "random_uniform_int/.*", + "random_uniform/.*", + "random_poisson_v2/.*" + }; + + /* As per TFGraphTestList.printArraysDebugging - this field defines a set of regexes for test cases that should have + all arrays printed during execution. + If a test name matches any regex here, an ExecPrintListener will be added to the listeners, and all output + arrays will be printed during execution + */ + private final List debugModeRegexes = Arrays.asList("fused_batch_norm/float16_nhwc"); + + + + private ExtensionContext.Store getStore(ExtensionContext context) { + return context.getStore(ExtensionContext.Namespace.create(getClass(), context.getRequiredTestMethod())); + } + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) { + String modelName = getStore(extensionContext).toString(); + if(EXECUTE_ONLY_MODELS.isEmpty()) { + for(String s : IGNORE_REGEXES) { + if(modelName.matches(s)) { + log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s); + assumeFalse(true); + } + } + } else if(!EXECUTE_ONLY_MODELS.contains(modelName)) { + log.info("Not executing " + modelName); + assumeFalse(true); + //OpValidationSuite.ignoreFailing(); + } + + + return ConditionEvaluationResult.disabled("Method found"); + } +} diff --git a/nd4j/samediff-import/samediff-import-tensorflow/variables-added-new.txt b/nd4j/samediff-import/samediff-import-tensorflow/variables-added-new.txt index a9255544b..0febb312a 100644 --- a/nd4j/samediff-import/samediff-import-tensorflow/variables-added-new.txt +++ b/nd4j/samediff-import/samediff-import-tensorflow/variables-added-new.txt @@ -1,23 +1,4 @@ -Reshape,Reshape -Lenet/conv1_1/Conv2D,Lenet/conv1_1/Conv2D -Lenet/conv1_1/BiasAdd,Lenet/conv1_1/BiasAdd -Lenet/conv1_1/Relu,Lenet/conv1_1/Relu -Lenet/pool2_1/MaxPool,Lenet/pool2_1/MaxPool -Lenet/conv3_1/Conv2D,Lenet/conv3_1/Conv2D -Lenet/conv3_1/BiasAdd,Lenet/conv3_1/BiasAdd -Lenet/conv3_1/Relu,Lenet/conv3_1/Relu -Lenet/pool4_1/MaxPool,Lenet/pool4_1/MaxPool -Lenet/conv5_1/Conv2D,Lenet/conv5_1/Conv2D -Lenet/conv5_1/BiasAdd,Lenet/conv5_1/BiasAdd -Lenet/conv5_1/Relu,Lenet/conv5_1/Relu -Lenet/flat6_1/flatten/Shape,Lenet/flat6_1/flatten/Shape -Lenet/flat6_1/flatten/strided_slice,Lenet/flat6_1/flatten/strided_slice -Lenet/flat6_1/flatten/Reshape/shape,Lenet/flat6_1/flatten/Reshape/shape -Lenet/flat6_1/flatten/Reshape,Lenet/flat6_1/flatten/Reshape -Lenet/fc7_1/MatMul,Lenet/fc7_1/MatMul -Lenet/fc7_1/BiasAdd,Lenet/fc7_1/BiasAdd -Lenet/fc7_1/Relu,Lenet/fc7_1/Relu -Lenet/fc9_1/MatMul,Lenet/fc9_1/MatMul -Lenet/fc9_1/BiasAdd,Lenet/fc9_1/BiasAdd -Lenet/fc9_1/Relu,Lenet/fc9_1/Relu -output,output +in_0/read,in_0/read +in_1/read,in_1/read +Add,Add +Zeta,Zeta