More test fixes

Signed-off-by: brian <brian@brutex.de>
master
Brian Rosenberger 2022-10-11 13:52:52 +02:00
parent 011ce913c9
commit a279242459
24 changed files with 149 additions and 33 deletions

View File

@ -20,13 +20,15 @@
package org.datavec.api.util.files; package org.datavec.api.util.files;
import lombok.NonNull;
import java.io.File; import java.io.File;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
public class URIUtil { public class URIUtil {
public static URI fileToURI(File f) { public static URI fileToURI(@NonNull File f) {
try { try {
// manually construct URI (this is faster) // manually construct URI (this is faster)
String sp = slashify(f.getAbsoluteFile().getPath(), false); String sp = slashify(f.getAbsoluteFile().getPath(), false);

View File

@ -23,7 +23,7 @@ apply from: "${project.rootProject.projectDir}/createTestBackends.gradle"
dependencies { dependencies {
implementation "com.codepoetics:protonpack:1.15" implementation "com.codepoetics:protonpack:1.15"
implementation projects.cavisDatavec.cavisDatavecApi implementation projects.cavisDatavec.cavisDatavecApi
implementation projects.cavisDatavec.cavisDatavecArrow implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataArrow
implementation projects.cavisDnn.cavisDnnApi implementation projects.cavisDnn.cavisDnnApi
implementation "com.google.guava:guava" implementation "com.google.guava:guava"

View File

@ -40,6 +40,8 @@ public class DL4JSystemProperties {
*/ */
public static final String DL4J_RESOURCES_DIR_PROPERTY = "org.deeplearning4j.resources.directory"; public static final String DL4J_RESOURCES_DIR_PROPERTY = "org.deeplearning4j.resources.directory";
public static final String DISABLE_HELPER_PROPERTY = "org.deeplearning4j.disablehelperloading";
public static final String HELPER_DISABLE_DEFAULT_VALUE = "false";
/** /**
* Applicability: Numerous modules, including deeplearning4j-datasets and deeplearning4j-zoo<br> * Applicability: Numerous modules, including deeplearning4j-datasets and deeplearning4j-zoo<br>
* Description: Used to set the base URL for hosting of resources such as datasets (like MNIST) and pretrained * Description: Used to set the base URL for hosting of resources such as datasets (like MNIST) and pretrained

View File

@ -18,6 +18,7 @@
* ***************************************************************************** * *****************************************************************************
* *
*/ */
apply from: "${project.rootProject.projectDir}/createTestBackends.gradle"
dependencies { dependencies {
implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators

View File

@ -0,0 +1,116 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.nn.layers;
import lombok.extern.slf4j.Slf4j;
import org.deeplearning4j.common.config.DL4JClassLoading;
import org.nd4j.linalg.factory.Nd4j;
import static org.deeplearning4j.common.config.DL4JSystemProperties.DISABLE_HELPER_PROPERTY;
import static org.deeplearning4j.common.config.DL4JSystemProperties.HELPER_DISABLE_DEFAULT_VALUE;
/**
* Simple meta helper util class for instantiating
* platform specific layer helpers that handle interaction with
* lower level libraries like cudnn and onednn.
*
* @author Adam Gibson
*/
@Slf4j
public class HelperUtils {
/**
* Creates a {@link LayerHelper}
* for use with platform specific code.
* @param <T> the actual class type to be returned
* @param cudnnHelperClassName the cudnn class name
* @param oneDnnClassName the one dnn class name
* @param layerHelperSuperClass the layer helper super class
* @param layerName the name of the layer to be created
* @param arguments the arguments to be used in creation of the layer
* @return
*/
public static <T extends LayerHelper> T createHelper(String cudnnHelperClassName,
String oneDnnClassName,
Class<? extends LayerHelper> layerHelperSuperClass,
String layerName,
Object... arguments) {
Boolean disabled = Boolean.parseBoolean(System.getProperty(DISABLE_HELPER_PROPERTY,HELPER_DISABLE_DEFAULT_VALUE));
if(disabled) {
System.out.println("Disabled helper creation, returning null");
return null;
}
String backend = Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend");
LayerHelper helperRet = null;
if("CUDA".equalsIgnoreCase(backend) && cudnnHelperClassName != null && !cudnnHelperClassName.isEmpty()) {
if(DL4JClassLoading.loadClassByName(cudnnHelperClassName) != null) {
log.debug("Attempting to initialize cudnn helper {}",cudnnHelperClassName);
helperRet = (LayerHelper) DL4JClassLoading.<LayerHelper>createNewInstance(
cudnnHelperClassName,
(Class<? super LayerHelper>) layerHelperSuperClass,
new Object[]{arguments});
log.debug("Cudnn helper {} successfully initialized",cudnnHelperClassName);
}
else {
log.warn("Unable to find class {} using the classloader set for Dl4jClassLoading. Trying to use class loader that loaded the class {} instead.",cudnnHelperClassName,layerHelperSuperClass.getName());
ClassLoader classLoader = DL4JClassLoading.getDl4jClassloader();
DL4JClassLoading.setDl4jClassloaderFromClass(layerHelperSuperClass);
try {
helperRet = (LayerHelper) DL4JClassLoading.<LayerHelper>createNewInstance(
cudnnHelperClassName,
(Class<? super LayerHelper>) layerHelperSuperClass,
arguments);
} catch (Exception e) {
log.warn("Unable to use helper implementation {} for helper type {}, please check your classpath. Falling back to built in normal methods for now.",cudnnHelperClassName,layerHelperSuperClass.getName());
}
log.warn("Returning class loader to original one.");
DL4JClassLoading.setDl4jClassloader(classLoader);
}
if (helperRet != null && !helperRet.checkSupported()) {
return null;
}
if(helperRet != null) {
log.debug("{} successfully initialized",cudnnHelperClassName);
}
} else if("CPU".equalsIgnoreCase(backend) && oneDnnClassName != null && !oneDnnClassName.isEmpty()) {
helperRet = DL4JClassLoading.<LayerHelper>createNewInstance(
oneDnnClassName,
arguments);
log.trace("Created oneDNN helper: {}, layer {}", oneDnnClassName,layerName);
}
if (helperRet != null && !helperRet.checkSupported()) {
log.debug("Removed helper {} as not supported", helperRet.getClass());
return null;
}
return (T) helperRet;
}
}

View File

@ -37,4 +37,6 @@ public interface LayerHelper {
*/ */
Map<String,Long> helperMemoryUse(); Map<String,Long> helperMemoryUse();
boolean checkSupported();
} }

View File

@ -59,4 +59,8 @@ public class BaseMKLDNNHelper {
} }
} }
public boolean checkSupported() {
return mklDnnEnabled();
}
} }

View File

@ -197,4 +197,9 @@ public class MKLDNNBatchNormHelper implements BatchNormalizationHelper {
public Map<String, Long> helperMemoryUse() { public Map<String, Long> helperMemoryUse() {
return Collections.emptyMap(); return Collections.emptyMap();
} }
@Override
public boolean checkSupported() {
return BaseMKLDNNHelper.mklDnnEnabled();
}
} }

View File

@ -30,6 +30,7 @@ import org.deeplearning4j.nn.workspace.ArrayType;
import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr;
import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.activations.IActivation;
import org.nd4j.linalg.activations.impl.*; import org.nd4j.linalg.activations.impl.*;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.DynamicCustomOp;
import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.linalg.api.shape.LongShapeDescriptor;
@ -43,6 +44,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
public class MKLDNNLSTMHelper implements LSTMHelper { public class MKLDNNLSTMHelper implements LSTMHelper {
public MKLDNNLSTMHelper(DataType dataType) {}
@Override @Override
public boolean checkSupported(IActivation gateActivationFn, IActivation activationFn, boolean hasPeepholeConnections) { public boolean checkSupported(IActivation gateActivationFn, IActivation activationFn, boolean hasPeepholeConnections) {
//TODO check other activation functions for MKLDNN //TODO check other activation functions for MKLDNN
@ -159,6 +161,11 @@ public class MKLDNNLSTMHelper implements LSTMHelper {
return Collections.emptyMap(); return Collections.emptyMap();
} }
@Override
public boolean checkSupported() {
return BaseMKLDNNHelper.mklDnnEnabled();
}
private int activationToArg(IActivation a){ private int activationToArg(IActivation a){
//0=tanh, 1=relu, 2=sigmoid, 3=affine, 4=leaky relu, 5= thresholded relu, 6=scaled tanh, 7=hard sigmoid, 8=ELU, 9=softsign, 10=softplus //0=tanh, 1=relu, 2=sigmoid, 3=affine, 4=leaky relu, 5= thresholded relu, 6=scaled tanh, 7=hard sigmoid, 8=ELU, 9=softsign, 10=softplus
if(a instanceof ActivationTanH) if(a instanceof ActivationTanH)

View File

@ -94,4 +94,5 @@ public class MKLDNNLocalResponseNormalizationHelper extends BaseMKLDNNHelper imp
public Map<String, Long> helperMemoryUse() { public Map<String, Long> helperMemoryUse() {
return Collections.emptyMap(); return Collections.emptyMap();
} }
} }

View File

@ -592,6 +592,11 @@ public class BidirectionalLayer implements RecurrentLayer {
} }
return ret; return ret;
} }
@Override
public boolean checkSupported() {
return true;
}
} }
@Override @Override

View File

@ -20,50 +20,21 @@
package org.deeplearning4j.nn.layers; package org.deeplearning4j.nn.layers;
import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.layers.ActivationLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.conf.layers.*;
import org.deeplearning4j.nn.graph.ComputationGraph;
import org.deeplearning4j.nn.layers.convolution.ConvolutionHelper; import org.deeplearning4j.nn.layers.convolution.ConvolutionHelper;
import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingHelper; import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingHelper;
import org.deeplearning4j.nn.layers.mkldnn.*; import org.deeplearning4j.nn.layers.mkldnn.*;
import org.deeplearning4j.nn.layers.normalization.BatchNormalizationHelper; import org.deeplearning4j.nn.layers.normalization.BatchNormalizationHelper;
import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalizationHelper; import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalizationHelper;
import org.deeplearning4j.nn.layers.recurrent.LSTMHelper; import org.deeplearning4j.nn.layers.recurrent.LSTMHelper;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.activations.impl.ActivationELU;
import org.nd4j.linalg.activations.impl.ActivationRationalTanh;
import org.nd4j.linalg.activations.impl.ActivationSoftmax;
import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import java.util.List; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.*;
/** /**
*/ */
@DisplayName("Activation Layer Test") @DisplayName("Activation Layer Test")
@NativeTag
@Tag(TagNames.CUSTOM_FUNCTIONALITY)
@Tag(TagNames.DL4J_OLD_API)
public class HelperUtilsTest extends BaseDL4JTest { public class HelperUtilsTest extends BaseDL4JTest {
@Override @Override

View File

@ -116,8 +116,8 @@ include ':cavis-dnn:cavis-dnn-spark:cavis-dnn-spark-parameterserver'
include ':cavis-dnn:cavis-dnn-tsne' include ':cavis-dnn:cavis-dnn-tsne'
include ':cavis-datavec' include ':cavis-datavec'
include ':cavis-datavec:cavis-datavec-api' include ':cavis-datavec:cavis-datavec-api'
include ':cavis-datavec:cavis-datavec-arrow'
include ':cavis-datavec:cavis-datavec-data' include ':cavis-datavec:cavis-datavec-data'
include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-arrow'
include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-image' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-image'
include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-audio' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-audio'
include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-codec' include ':cavis-datavec:cavis-datavec-data:cavis-datavec-data-codec'