Brutex/cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add large test tags, ensure that small runs finish, get rid of test timeouts

Browse Source
This commit is contained in:
agibsonccc 2021-03-25 23:22:18 +09:00
parent 652b854083
commit 3e60302e8c
104 changed files with 896 additions and 667 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/cpu-sanity-check-tests.yaml vendored
View File

@ -31,7 +31,7 @@ jobs:
protoc --version
cd dl4j-test-resources-master && mvn clean install -DskipTests && cd ..
export OMP_NUM_THREADS=1
mvn -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -Dlibnd4j.buildthreads=1 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test
mvn -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DexcludedGroups="long-running-tests,large-resources" -Dlibnd4j.buildthreads=1 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test --fail-never
windows-x86_64:
runs-on: windows-2019
@ -44,7 +44,7 @@ jobs:
run: |
set "PATH=C:\msys64\usr\bin;%PATH%"
export OMP_NUM_THREADS=1
mvn -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DskipTestResourceEnforcement=true -Ptestresources -Dlibnd4j.buildthreads=1 -Dlibnd4j.build="Debug" -Djavacpp.platform=windows-x86_64 -libnd4j.platform=windows-x86_64 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test
mvn -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DexcludedGroups="long-running-tests,large-resources" -DskipTestResourceEnforcement=true -Ptestresources -Dlibnd4j.buildthreads=1 -Dlibnd4j.build="Debug" -Djavacpp.platform=windows-x86_64 -libnd4j.platform=windows-x86_64 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test --fail-never

3
.github/workflows/run-cpu-integration-tests-self-hosted.yml vendored
View File

@ -22,5 +22,6 @@ jobs:
cmake --version
protoc --version
export OMP_NUM_THREADS=1
mvn -DexcludedGroups=long-running-tests -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cpu clean test
mvn -DexcludedGroups="long-running-tests,large-resources" -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cpu clean test
mvn -Ptestresources -Pnd4j-tests-cpu -Dtest.offheap.size=14g -Dtest.heap.size=6g clean test

2
.github/workflows/run-cpu-tests-sanity-checks.yml vendored
View File

@ -34,5 +34,5 @@ jobs:
cmake --version
protoc --version
export OMP_NUM_THREADS=1
mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -Pnd4j-tests-cpu --also-make clean test
mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DexcludedGroups="long-running-tests,large-resources" -Pnd4j-tests-cpu --also-make clean test --fail-never

3
.github/workflows/run-gpu-integration-tests-self-hosted.yml vendored
View File

@ -34,5 +34,6 @@ jobs:
cmake --version
protoc --version
export OMP_NUM_THREADS=1
mvn -DexcludedGroups=long-running-tests -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cuda clean test
mvn -DexcludedGroups="long-running-tests,large-resources" -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cuda clean test --fail-never
mvn -Ptestresources -Pnd4j-tests-cuda -Dtest.offheap.size=14g -Dtest.heap.size=6g clean test --fail-never

2
.github/workflows/run-gpu-tests-sanity-checks.yml vendored
View File

@ -35,5 +35,5 @@ jobs:
protoc --version
bash ./change-cuda-versions.sh 11.2
export OMP_NUM_THREADS=1
mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-cuda-11.2,:samediff-import,:libnd4j" -Dlibnd4j.helper=cudnn -Ptest-nd4j-cuda --also-make -Dlibnd4j.chip=cuda clean test
mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-cuda-11.2,:samediff-import,:libnd4j" -Dlibnd4j.helper=cudnn -Ptest-nd4j-cuda --also-make -Dlibnd4j.chip=cuda clean test --fail-never

11
ADRs/0006 - Test architecture.md
View File

@ -42,6 +42,17 @@ A few kinds of tags exist:
7. RNG: (rng) for RNG related tests
8. Samediff:(samediff) samediff related tests
9. Training related functionality
10. long-running-tests: The longer running tests that take a longer execution time
11. large-resources: tests requiring a large amount of ram/cpu (>= 2g up to 16g)
New maven properties for maven surefire:
test.offheap.size: tunes off heap size for javacpp
test.heap.size: tunes heap size of test jvms
Auto tuning the number of CPU cores for tests relative to the number of CPUs present
## Consequences

4
datavec/datavec-arrow/src/main/java/org/datavec/arrow/ArrowConverter.java
View File

@ -58,6 +58,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.*;
@ -171,7 +172,8 @@ public class ArrowConverter {
ByteBuffer direct = ByteBuffer.allocateDirect(fieldVector.getDataBuffer().capacity());
direct.order(ByteOrder.nativeOrder());
fieldVector.getDataBuffer().getBytes(0,direct);
direct.rewind();
Buffer buffer1 = (Buffer) direct;
buffer1.rewind();
switch(type) {
case Integer:
buffer = Nd4j.createBuffer(direct, DataType.INT,cols,0);

1
datavec/datavec-data/datavec-data-image/pom.xml
View File

@ -119,6 +119,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<classpathDependencyExcludes>
<classpathDependencyExclude>com.google.android:android
</classpathDependencyExclude>

BIN
datavec/datavec-jdbc/datavecTests/db.lck
View File

Binary file not shown.

BIN
datavec/datavec-jdbc/datavecTests/log/log.ctrl
View File

Binary file not shown.

23
deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java
View File

@ -19,31 +19,26 @@
*/
package org.deeplearning4j.datasets;
import org.apache.commons.io.FileUtils;
import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.datasets.base.MnistFetcher;
import org.deeplearning4j.common.resources.DL4JResources;
import org.deeplearning4j.datasets.base.MnistFetcher;
import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.io.TempDir;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.conditions.Conditions;
import java.io.File;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.junit.jupiter.api.Assertions.*;
@DisplayName("Mnist Fetcher Test")
@NativeTag
@ -65,6 +60,9 @@ class MnistFetcherTest extends BaseDL4JTest {
@Test
@DisplayName("Test Mnist")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.FILE_IO)
void testMnist() throws Exception {
MnistDataSetIterator iter = new MnistDataSetIterator(32, 60000, false, true, false, -1);
int count = 0;
@ -91,6 +89,9 @@ class MnistFetcherTest extends BaseDL4JTest {
@Test
@DisplayName("Test Mnist Data Fetcher")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.FILE_IO)
void testMnistDataFetcher() throws Exception {
MnistFetcher mnistFetcher = new MnistFetcher();
File mnistDir = mnistFetcher.downloadAndUntar();
@ -99,6 +100,9 @@ class MnistFetcherTest extends BaseDL4JTest {
}
@Test
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.FILE_IO)
public void testMnistSubset() throws Exception {
final int numExamples = 100;
MnistDataSetIterator iter1 = new MnistDataSetIterator(10, numExamples, false, true, true, 123);
@ -144,6 +148,9 @@ class MnistFetcherTest extends BaseDL4JTest {
@Test
@DisplayName("Test Subset Repeatability")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.FILE_IO)
void testSubsetRepeatability() throws Exception {
MnistDataSetIterator it = new MnistDataSetIterator(1, 1, false, false, true, 0);
DataSet d1 = it.next();

1
deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java
View File

@ -51,6 +51,7 @@ public class TestEmnistDataSetIterator extends BaseDL4JTest {
@Test
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testEmnistDataSetIterator() throws Exception {
int batchSize = 128;

18
deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java
View File

@ -63,6 +63,8 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.deeplearning4j.util.ModelSerializer;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.activations.Activation;
@ -1717,8 +1719,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest {
MultiLayerTest.CheckModelsListener listener = new MultiLayerTest.CheckModelsListener();
net.setListeners(listener);
INDArray f = Nd4j.create(1,10);
INDArray l = Nd4j.create(1,10);
INDArray f = Nd4j.create(DataType.DOUBLE,1,10);
INDArray l = Nd4j.create(DataType.DOUBLE,1,10);
DataSet ds = new DataSet(f,l);
MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(f,l);
@ -2117,9 +2119,10 @@ public class TestComputationGraphNetwork extends BaseDL4JTest {
}
@Test
@Execution(ExecutionMode.SAME_THREAD)
@Tag(TagNames.NEEDS_VERIFY)
@Disabled
public void testCompGraphInputReuse() {
Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE);
int inputSize = 5;
int outputSize = 6;
int layerSize = 3;
@ -2134,7 +2137,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest {
.setOutputs("out")
.addLayer("0",new DenseLayer.Builder().nIn(inputSize).nOut(layerSize).build(),"in")
.addVertex("combine", new MergeVertex(), "0", "0", "0")
.addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(3*layerSize).nOut(outputSize)
.addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(3*layerSize)
.nOut(outputSize)
.activation(Activation.SIGMOID).build(),"combine")
.build();
@ -2143,8 +2147,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest {
int dataSize = 11;
INDArray features = Nd4j.rand(new int[] {dataSize, inputSize});
INDArray labels = Nd4j.rand(new int[] {dataSize, outputSize});
INDArray features = Nd4j.rand(DataType.DOUBLE,new int[] {dataSize, inputSize});
INDArray labels = Nd4j.rand(DataType.DOUBLE,new int[] {dataSize, outputSize});
boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{features})
.labels(new INDArray[]{labels}));

27
deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java
View File

@ -23,8 +23,11 @@ import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.nn.conf.distribution.*;
import org.deeplearning4j.nn.conf.serde.JsonMappers;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.rng.Random;
import org.nd4j.linalg.factory.Nd4j;
@ -69,14 +72,19 @@ class LegacyWeightInitTest extends BaseDL4JTest {
final long[] shape = { 5, 5 };
final long fanIn = shape[0];
final long fanOut = shape[1];
final INDArray inLegacy = Nd4j.create(fanIn * fanOut);
final INDArray inLegacy = Nd4j.create(DataType.DOUBLE,fanIn * fanOut);
final INDArray inTest = inLegacy.dup();
for (WeightInit legacyWi : WeightInit.values()) {
if (legacyWi != WeightInit.DISTRIBUTION) {
Nd4j.getRandom().setSeed(SEED);
final INDArray expected = WeightInitUtil.initWeights(fanIn, fanOut, shape, legacyWi, null, inLegacy);
final INDArray expected = WeightInitUtil.
initWeights(fanIn, fanOut, shape, legacyWi, null, inLegacy)
.castTo(DataType.DOUBLE);
Nd4j.getRandom().setSeed(SEED);
final INDArray actual = legacyWi.getWeightInitFunction().init(fanIn, fanOut, shape, WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, inTest);
final INDArray actual = legacyWi.getWeightInitFunction()
.init(fanIn, fanOut, shape,
WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, inTest)
.castTo(DataType.DOUBLE);
assertArrayEquals(shape, actual.shape(),"Incorrect shape for " + legacyWi + "!");
assertEquals( expected, actual,"Incorrect weight initialization for " + legacyWi + "!");
}
@ -88,17 +96,24 @@ class LegacyWeightInitTest extends BaseDL4JTest {
*/
@Test
@DisplayName("Init Params From Distribution")
@Execution(ExecutionMode.SAME_THREAD)
@Disabled(TagNames.NEEDS_VERIFY)
void initParamsFromDistribution() {
// To make identity happy
final long[] shape = { 3, 7 };
final long fanIn = shape[0];
final long fanOut = shape[1];
final INDArray inLegacy = Nd4j.create(fanIn * fanOut);
final INDArray inLegacy = Nd4j.create(DataType.DOUBLE,fanIn * fanOut);
final INDArray inTest = inLegacy.dup();
for (Distribution dist : distributions) {
Nd4j.getRandom().setSeed(SEED);
final INDArray expected = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.DISTRIBUTION, Distributions.createDistribution(dist), inLegacy);
final INDArray actual = new WeightInitDistribution(dist).init(fanIn, fanOut, shape, WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, inTest);
final INDArray expected = WeightInitUtil
.initWeights(fanIn, fanOut, shape, WeightInit.DISTRIBUTION,
Distributions.createDistribution(dist), inLegacy)
.castTo(DataType.DOUBLE);
final INDArray actual = new WeightInitDistribution(dist)
.init(fanIn, fanOut, shape, WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER,
inTest).castTo(DataType.DOUBLE);
assertArrayEquals(shape, actual.shape(),"Incorrect shape for " + dist.getClass().getSimpleName() + "!");
assertEquals( expected, actual,"Incorrect weight initialization for " + dist.getClass().getSimpleName() + "!");
}

11
deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java
View File

@ -34,6 +34,8 @@ import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.activations.Activation;
@ -56,14 +58,14 @@ public class RandomTests extends BaseDL4JTest {
*
* @throws Exception
*/
@Test
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
@Execution(ExecutionMode.SAME_THREAD)
public void testModelInitialParamsEquality1() throws Exception {
final List<Model> models = new CopyOnWriteArrayList<>();
for (int i = 0; i < 4; i++) {
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
Thread thread = new Thread(() -> {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(119) // Training iterations as above
.l2(0.0005)
//.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
@ -91,7 +93,6 @@ public class RandomTests extends BaseDL4JTest {
network.init();
models.add(network);
}
});
thread.start();

3
deeplearning4j/deeplearning4j-dataimport-solrj/pom.xml
View File

@ -47,8 +47,9 @@
<configuration>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<argLine>-Ddtype=float -Dfile.encoding=UTF-8
-Dtest.solr.allowed.securerandom=NativePRNG
-Dtest.solr.allowed.securerandom=NativePRNG -Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}
</argLine>
<includes>
<!-- Default setting only runs tests that start/end with "Test" -->

2
deeplearning4j/deeplearning4j-dataimport-solrj/src/test/java/org/deeplearning4j/nn/dataimport/solr/client/solrj/io/stream/TupleStreamDataSetIteratorTest.java
View File

@ -48,6 +48,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
@DisplayName("Tuple Stream Data Set Iterator Test")
@Tag(TagNames.SOLR)
@Tag(TagNames.DIST_SYSTEMS)
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
class TupleStreamDataSetIteratorTest extends SolrCloudTestCase {
static {

3
deeplearning4j/deeplearning4j-modelexport-solr/pom.xml
View File

@ -41,7 +41,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-Ddtype=float -Dfile.encoding=UTF-8 -Xmx8g
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<argLine>-Ddtype=float -Dfile.encoding=UTF-8 -Xmx${test.heap.size}
-Dtest.solr.allowed.securerandom=NativePRNG
</argLine>
<includes>

2
deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/SequenceVectorsTest.java
View File

@ -76,6 +76,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Tag(TagNames.FILE_IO)
@NativeTag
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public class SequenceVectorsTest extends BaseDL4JTest {
protected static final Logger logger = LoggerFactory.getLogger(SequenceVectorsTest.class);

7
deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java
View File

@ -424,12 +424,7 @@ public class GradientCheckUtil {
throw new IllegalArgumentException(
"Invalid labels arrays: expect " + c.net.getNumOutputArrays() + " outputs");
DataType dataType = DataTypeUtil.getDtypeFromContext();
if (dataType != DataType.DOUBLE) {
throw new IllegalStateException("Cannot perform gradient check: Datatype is not set to double precision ("
+ "is: " + dataType + "). Double precision must be used for gradient checks. Set "
+ "DataTypeUtil.setDTypeForContext(DataType.DOUBLE); before using GradientCheckUtil");
}
DataType netDataType = c.net.getConfiguration().getDataType();
if (netDataType != DataType.DOUBLE) {

27
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectorsTest.java
View File

@ -21,6 +21,8 @@
package org.deeplearning4j.spark.models.sequencevectors;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@ -35,9 +37,12 @@ import org.deeplearning4j.spark.models.word2vec.SparkWord2VecTest;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.junit.jupiter.api.*;
import org.nd4j.common.primitives.Counter;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@ -47,6 +52,7 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals;
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Slf4j
public class SparkSequenceVectorsTest extends BaseDL4JTest {
@Override
@ -57,6 +63,27 @@ public class SparkSequenceVectorsTest extends BaseDL4JTest {
protected static List<Sequence<VocabWord>> sequencesCyclic;
private JavaSparkContext sc;
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@BeforeEach
public void setUp() throws Exception {
if (sequencesCyclic == null) {

28
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/word2vec/SparkWord2VecTest.java
View File

@ -20,6 +20,9 @@
package org.deeplearning4j.spark.models.word2vec;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@ -35,11 +38,14 @@ import org.deeplearning4j.spark.models.sequencevectors.export.SparkModelExporter
import org.deeplearning4j.spark.models.sequencevectors.learning.elements.SparkSkipGram;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.junit.jupiter.api.*;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.parameterserver.distributed.conf.VoidConfiguration;
import java.io.File;
import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@ -48,6 +54,7 @@ import static org.junit.jupiter.api.Assertions.*;
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Slf4j
public class SparkWord2VecTest extends BaseDL4JTest {
@Override
@ -58,6 +65,27 @@ public class SparkWord2VecTest extends BaseDL4JTest {
private static List<String> sentences;
private JavaSparkContext sc;
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@BeforeEach
public void setUp() throws Exception {
if (sentences == null) {

33
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java
View File

@ -21,11 +21,15 @@
package org.deeplearning4j.spark.models.embeddings.word2vec;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.deeplearning4j.common.resources.DL4JResources;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.io.TempDir;
import org.nd4j.common.io.ClassPathResource;
@ -41,11 +45,14 @@ import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFac
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.resources.strumpf.StrumpfResolver;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.ndarray.INDArray;
import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
@ -53,21 +60,37 @@ import java.util.Collection;
import static org.junit.jupiter.api.Assertions.*;
@Disabled
@Tag(TagNames.FILE_IO)
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Slf4j
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class Word2VecTest {
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@Test
public void testConcepts(@TempDir Path testDir) throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
// These are all default values for word2vec
SparkConf sparkConf = new SparkConf().setMaster("local[8]")
.set("spark.driver.host", "localhost")

31
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java
View File

@ -20,21 +20,50 @@
package org.deeplearning4j.spark.text;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.spark.models.embeddings.word2vec.Word2VecVariables;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.nd4j.common.resources.Downloader;
import java.io.File;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.net.URI;
import java.util.Collections;
import java.util.Map;
@Slf4j
public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable {
protected transient JavaSparkContext sc;
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@Override
public long getTimeoutMilliseconds() {
return 120000L;

33
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java
View File

@ -21,6 +21,7 @@
package org.deeplearning4j.spark.text;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
@ -35,10 +36,8 @@ import org.deeplearning4j.spark.models.embeddings.word2vec.Word2Vec;
import org.deeplearning4j.spark.text.functions.CountCumSum;
import org.deeplearning4j.spark.text.functions.TextPipeline;
import org.deeplearning4j.text.stopwords.StopWords;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.*;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.ndarray.INDArray;
@ -48,6 +47,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
import java.io.File;
import java.net.URI;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
@ -74,6 +75,26 @@ public class TextPipelineTest extends BaseSparkTest {
return sc.parallelize(sentenceList, 2);
}
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@BeforeEach
public void before() throws Exception {
conf = new SparkConf().setMaster("local[4]").setAppName("sparktest").set("spark.driver.host", "localhost");
@ -102,10 +123,6 @@ public class TextPipelineTest extends BaseSparkTest {
@Test
public void testTokenizer() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
JavaSparkContext sc = getContext();
JavaRDD<String> corpusRDD = getCorpusRDD(sc);
Broadcast<Map<String, Object>> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap());

30
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java
View File

@ -20,6 +20,9 @@
package org.deeplearning4j.spark.parameterserver;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@ -29,7 +32,9 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer;
import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.nd4j.common.resources.Downloader;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
@ -37,12 +42,14 @@ import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.learning.config.Nesterovs;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import java.io.File;
import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@Slf4j
public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable {
protected transient JavaSparkContext sc;
protected transient INDArray labels;
@ -60,6 +67,27 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable
return 120000L;
}
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@BeforeEach
public void before() {

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java
View File

@ -40,10 +40,6 @@ public class SharedTrainingAccumulationFunctionTest {
@Test
public void testAccumulation1() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
INDArray updates1 = Nd4j.create(1000).assign(1.0);
INDArray updates2 = Nd4j.create(1000).assign(2.0);
INDArray expUpdates = Nd4j.create(1000).assign(3.0);

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java
View File

@ -43,10 +43,6 @@ public class SharedTrainingAggregateFunctionTest {
@Test
public void testAggregate1() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
INDArray updates1 = Nd4j.create(1000).assign(1.0);
INDArray updates2 = Nd4j.create(1000).assign(2.0);
INDArray expUpdates = Nd4j.create(1000).assign(3.0);

32
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java
View File

@ -21,15 +21,21 @@
package org.deeplearning4j.spark.parameterserver.iterators;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@ -39,17 +45,35 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Slf4j
public class VirtualDataSetIteratorTest {
@BeforeEach
public void setUp() throws Exception {}
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@Test
public void testSimple1() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
List<Iterator<DataSet>> iterators = new ArrayList<>();
List<DataSet> first = new ArrayList<>();

32
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java
View File

@ -21,12 +21,18 @@
package org.deeplearning4j.spark.parameterserver.iterators;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@ -35,18 +41,36 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Slf4j
public class VirtualIteratorTest {
@BeforeEach
public void setUp() throws Exception {
//
}
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@Test
public void testIteration1() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
List<Integer> integers = new ArrayList<>();
for (int i = 0; i < 100; i++) {
integers.add(i);

31
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java
View File

@ -21,19 +21,24 @@
package org.deeplearning4j.spark.parameterserver.modelimport.elephas;
import com.sun.jna.Platform;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.api.java.JavaSparkContext;
import org.deeplearning4j.spark.impl.graph.SparkComputationGraph;
import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer;
import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster;
import org.deeplearning4j.spark.parameterserver.BaseSparkTest;
import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.nd4j.common.io.ClassPathResource;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
@ -43,14 +48,32 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Slf4j
public class TestElephasImport extends BaseSparkTest {
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@Test
public void testElephasSequentialImport() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
String modelPath = "modelimport/elephas/elephas_sequential.h5";
SparkDl4jMultiLayer model = importElephasSequential(sc, modelPath);
// System.out.println(model.getNetwork().summary());

11
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java
View File

@ -44,7 +44,6 @@ import org.deeplearning4j.spark.api.RDDTrainingApproach;
import org.deeplearning4j.spark.api.TrainingMaster;
import org.deeplearning4j.spark.impl.graph.SparkComputationGraph;
import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer;
import org.deeplearning4j.spark.parameterserver.BaseSparkTest;
import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster;
import org.junit.jupiter.api.Disabled;
@ -75,6 +74,7 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import static org.junit.jupiter.api.Assertions.*;
import org.deeplearning4j.spark.parameterserver.BaseSparkTest;
@Slf4j
//@Disabled("AB 2019/05/21 - Failing - Issue #7657")
@ -82,6 +82,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
@NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class GradientSharingTrainingTest extends BaseSparkTest {
@ -339,11 +341,12 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
}
@Test @Disabled
@Test
public void testEpochUpdating(@TempDir Path testDir) throws Exception {
//Ensure that epoch counter is incremented properly on the workers
File temp = testDir.toFile();
File temp = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile();
temp.mkdirs();
//TODO this probably won't work everywhere...
String controller = Inet4Address.getLocalHost().getHostAddress();
@ -394,7 +397,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
}
JavaRDD<String> pathRdd = sc.parallelize(paths);
for( int i=0; i<3; i++ ) {
for( int i = 0; i < 3; i++) {
ThresholdAlgorithm ta = tm.getThresholdAlgorithm();
sparkNet.fitPaths(pathRdd);
//Check also that threshold algorithm was updated/averaged

123
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java
View File

@ -1,123 +0,0 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.spark.iterator;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.TaskContext;
import org.apache.spark.TaskContextHelper;
import org.nd4j.linalg.dataset.AsyncDataSetIterator;
import org.nd4j.linalg.api.memory.MemoryWorkspace;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.callbacks.DataSetCallback;
import org.nd4j.linalg.dataset.callbacks.DefaultCallback;
import org.nd4j.linalg.factory.Nd4j;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@Slf4j
public class SparkADSI extends AsyncDataSetIterator {
protected TaskContext context;
protected SparkADSI() {
super();
}
public SparkADSI(DataSetIterator baseIterator) {
this(baseIterator, 8);
}
public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue) {
this(iterator, queueSize, queue, true);
}
public SparkADSI(DataSetIterator baseIterator, int queueSize) {
this(baseIterator, queueSize, new LinkedBlockingQueue<DataSet>(queueSize));
}
public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace) {
this(baseIterator, queueSize, new LinkedBlockingQueue<DataSet>(queueSize), useWorkspace);
}
public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) {
this(baseIterator, queueSize, new LinkedBlockingQueue<DataSet>(queueSize), useWorkspace, new DefaultCallback(),
deviceId);
}
public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, DataSetCallback callback) {
this(baseIterator, queueSize, new LinkedBlockingQueue<DataSet>(queueSize), useWorkspace, callback);
}
public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace) {
this(iterator, queueSize, queue, useWorkspace, new DefaultCallback());
}
public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace,
DataSetCallback callback) {
this(iterator, queueSize, queue, useWorkspace, callback, Nd4j.getAffinityManager().getDeviceForCurrentThread());
}
public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue<DataSet> queue, boolean useWorkspace,
DataSetCallback callback, Integer deviceId) {
this();
if (queueSize < 2)
queueSize = 2;
this.deviceId = deviceId;
this.callback = callback;
this.useWorkspace = useWorkspace;
this.buffer = queue;
this.prefetchSize = queueSize;
this.backedIterator = iterator;
this.workspaceId = "SADSI_ITER-" + java.util.UUID.randomUUID().toString();
if (iterator.resetSupported())
this.backedIterator.reset();
context = TaskContext.get();
this.thread = new SparkPrefetchThread(buffer, iterator, terminator, null, Nd4j.getAffinityManager().getDeviceForCurrentThread());
/**
* We want to ensure, that background thread will have the same thread->device affinity, as master thread
*/
thread.setDaemon(true);
thread.start();
}
@Override
protected void externalCall() {
TaskContextHelper.setTaskContext(context);
}
public class SparkPrefetchThread extends AsyncPrefetchThread {
protected SparkPrefetchThread(BlockingQueue<DataSet> queue, DataSetIterator iterator, DataSet terminator, MemoryWorkspace workspace, int deviceId) {
super(queue, iterator, terminator, workspace, deviceId);
}
}
}

118
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java
View File

@ -1,118 +0,0 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.deeplearning4j.spark.iterator;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.TaskContext;
import org.apache.spark.TaskContextHelper;
import org.nd4j.linalg.dataset.AsyncMultiDataSetIterator;
import org.nd4j.linalg.dataset.api.MultiDataSet;
import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
import org.nd4j.linalg.dataset.callbacks.DataSetCallback;
import org.nd4j.linalg.dataset.callbacks.DefaultCallback;
import org.nd4j.linalg.factory.Nd4j;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@Slf4j
public class SparkAMDSI extends AsyncMultiDataSetIterator {
protected TaskContext context;
protected SparkAMDSI() {
super();
}
public SparkAMDSI(MultiDataSetIterator baseIterator) {
this(baseIterator, 8);
}
public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue) {
this(iterator, queueSize, queue, true);
}
public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize) {
this(baseIterator, queueSize, new LinkedBlockingQueue<MultiDataSet>(queueSize));
}
public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace) {
this(baseIterator, queueSize, new LinkedBlockingQueue<MultiDataSet>(queueSize), useWorkspace);
}
public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) {
this(baseIterator, queueSize, new LinkedBlockingQueue<MultiDataSet>(queueSize), useWorkspace,
new DefaultCallback(), deviceId);
}
public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace,
DataSetCallback callback) {
this(baseIterator, queueSize, new LinkedBlockingQueue<MultiDataSet>(queueSize), useWorkspace, callback);
}
public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
boolean useWorkspace) {
this(iterator, queueSize, queue, useWorkspace, null);
}
public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
boolean useWorkspace, DataSetCallback callback) {
this(iterator, queueSize, queue, useWorkspace, callback, Nd4j.getAffinityManager().getDeviceForCurrentThread());
}
public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue<MultiDataSet> queue,
boolean useWorkspace, DataSetCallback callback, Integer deviceId) {
this();
if (queueSize < 2)
queueSize = 2;
this.callback = callback;
this.buffer = queue;
this.backedIterator = iterator;
this.useWorkspaces = useWorkspace;
this.prefetchSize = queueSize;
this.workspaceId = "SAMDSI_ITER-" + java.util.UUID.randomUUID().toString();
this.deviceId = deviceId;
if (iterator.resetSupported())
this.backedIterator.reset();
this.thread = new SparkPrefetchThread(buffer, iterator, terminator, Nd4j.getAffinityManager().getDeviceForCurrentThread());
context = TaskContext.get();
thread.setDaemon(true);
thread.start();
}
@Override
protected void externalCall() {
TaskContextHelper.setTaskContext(context);
}
protected class SparkPrefetchThread extends AsyncPrefetchThread {
protected SparkPrefetchThread(@NonNull BlockingQueue<MultiDataSet> queue, @NonNull MultiDataSetIterator iterator, @NonNull MultiDataSet terminator, int deviceId) {
super(queue, iterator, terminator, deviceId);
}
}
}

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java
View File

@ -64,10 +64,6 @@ public class TestEarlyStoppingSpark extends BaseSparkTest {
@Test
public void testEarlyStoppingIris() {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.updater(new Sgd()).weightInit(WeightInit.XAVIER).list()

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java
View File

@ -67,10 +67,6 @@ public class TestEarlyStoppingSparkCompGraph extends BaseSparkTest {
@Test
public void testEarlyStoppingIris() {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.updater(new Sgd()).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java
View File

@ -76,10 +76,6 @@ public class TestDataVecDataSetFunctions extends BaseSparkTest {
@Test
public void testDataVecDataSetFunction(@TempDir Path testDir) throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
JavaSparkContext sc = getContext();
File f = testDir.toFile();

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java
View File

@ -51,10 +51,6 @@ public class TestExport extends BaseSparkTest {
@Test
public void testBatchAndExportDataSetsFunction() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
String baseDir = System.getProperty("java.io.tmpdir");
baseDir = FilenameUtils.concat(baseDir, "dl4j_spark_testBatchAndExport/");
baseDir = baseDir.replaceAll("\\\\", "/");

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java
View File

@ -70,10 +70,6 @@ public class TestPreProcessedData extends BaseSparkTest {
@Test
public void testPreprocessedData() {
//Test _loading_ of preprocessed data
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
int dataSetObjSize = 5;
int batchSizePerExecutor = 10;

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java
View File

@ -52,10 +52,6 @@ public class TestCustomLayer extends BaseSparkTest {
@Test
public void testSparkWithCustomLayer() {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
//Basic test - checks whether exceptions etc are thrown with custom layers + spark
//Custom layers are tested more extensively in dl4j core
MultiLayerConfiguration conf =

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java
View File

@ -77,10 +77,6 @@ public class TestSparkDl4jMultiLayer extends BaseSparkTest {
@Test
public void testEvaluationSimple() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
Nd4j.getRandom().setSeed(12345);
for( int evalWorkers : new int[]{1, 4, 8}) {

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java
View File

@ -61,10 +61,6 @@ public class TestTrainingStatsCollection extends BaseSparkTest {
@Test
public void testStatsCollection() throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
int nWorkers = numExecutors();
JavaSparkContext sc = getContext();

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java
View File

@ -60,10 +60,6 @@ public class TestListeners extends BaseSparkTest {
@Test
public void testStatsCollection() {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
JavaSparkContext sc = getContext();
int nExecutors = numExecutors();

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java
View File

@ -54,10 +54,6 @@ public class TestRepartitioning extends BaseSparkTest {
@Test
public void testRepartitioning() {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
List<String> list = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
list.add(String.valueOf(i));

4
deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestValidation.java
View File

@ -52,10 +52,6 @@ public class TestValidation extends BaseSparkTest {
@Test
public void testDataSetValidation(@TempDir Path folder) throws Exception {
if(Platform.isWindows()) {
//Spark tests don't run on windows
return;
}
File f = folder.toFile();
for( int i = 0; i < 3; i++ ) {

8
deeplearning4j/deeplearning4j-ui-parent/pom.xml
View File

@ -38,11 +38,17 @@
<module>deeplearning4j-ui</module>
<module>deeplearning4j-ui-components</module>
<module>deeplearning4j-ui-model</module>
<module>deeplearning4j-ui-standalone</module>
<module>deeplearning4j-vertx</module>
</modules>
<profiles>
<profile>
<id>ui-jar</id>
<modules>
<module>deeplearning4j-ui-standalone</module>
</modules>
</profile>
<profile>
<id>nd4j-tests-cpu</id>
</profile>

1
deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/MiscTests.java
View File

@ -41,6 +41,7 @@ import java.io.File;
@Tag(TagNames.DL4J_OLD_API)
@NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class MiscTests extends BaseDL4JTest {
@Override

1
deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestDownload.java
View File

@ -51,6 +51,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
@Tag(TagNames.DL4J_OLD_API)
@NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class TestDownload extends BaseDL4JTest {
@TempDir
static Path sharedTempDir;

1
deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestImageNet.java
View File

@ -61,6 +61,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@Tag(TagNames.DL4J_OLD_API)
@NativeTag
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class TestImageNet extends BaseDL4JTest {
@Override

28
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java
View File

@ -2308,7 +2308,7 @@ public class Nd4j {
data2.add(readSplit(data));
}
float[][] fArr = new float[data2.size()][0];
for(int i=0; i<data2.size(); i++ ){
for(int i = 0; i < data2.size(); i++) {
fArr[i] = data2.get(i);
}
ret = Nd4j.createFromArray(fArr).castTo(dataType);
@ -2785,7 +2785,7 @@ public class Nd4j {
* @return the random ndarray with the specified shape
*/
public static INDArray rand(@NonNull int... shape) {
INDArray ret = createUninitialized(shape, order()).castTo(Nd4j.defaultFloatingPointType()); //INSTANCE.rand(shape, Nd4j.getRandom());
INDArray ret = createUninitialized(shape, order()); //INSTANCE.rand(shape, Nd4j.getRandom());
return rand(ret);
}
@ -2793,7 +2793,7 @@ public class Nd4j {
* See {@link #rand(int[])}
*/
public static INDArray rand(@NonNull long... shape) {
INDArray ret = createUninitialized(shape, order()).castTo(Nd4j.defaultFloatingPointType()); //INSTANCE.rand(shape, Nd4j.getRandom());
INDArray ret = createUninitialized(shape, order()); //INSTANCE.rand(shape, Nd4j.getRandom());
return rand(ret);
}
@ -2806,7 +2806,7 @@ public class Nd4j {
public static INDArray rand(@NonNull DataType dataType, @NonNull long... shape) {
Preconditions.checkArgument(dataType.isFPType(),
"Can't create a random array of a non-floating point data type");
INDArray ret = createUninitialized(dataType, shape, order()).castTo(Nd4j.defaultFloatingPointType()); //INSTANCE.rand(shape, Nd4j.getRandom());
INDArray ret = createUninitialized(dataType, shape, order()); //INSTANCE.rand(shape, Nd4j.getRandom());
return rand(ret);
}
@ -2820,7 +2820,7 @@ public class Nd4j {
* @return the random ndarray with the specified shape
*/
public static INDArray rand(char order, @NonNull int... shape) {
INDArray ret = Nd4j.createUninitialized(shape, order).castTo(Nd4j.defaultFloatingPointType()); //INSTANCE.rand(order, shape);
INDArray ret = Nd4j.createUninitialized(shape, order); //INSTANCE.rand(order, shape);
return rand(ret);
}
@ -2829,7 +2829,7 @@ public class Nd4j {
*/
@Deprecated
public static INDArray rand(@NonNull DataType dataType, int[] shape, char order) {
return rand(dataType, order, ArrayUtil.toLongArray(shape)).castTo(Nd4j.defaultFloatingPointType());
return rand(dataType, order, ArrayUtil.toLongArray(shape));
}
/**
@ -2837,7 +2837,7 @@ public class Nd4j {
*/
@Deprecated
public static INDArray rand(@NonNull DataType dataType, char order, @NonNull int... shape) {
return rand(dataType, order, ArrayUtil.toLongArray(shape)).castTo(Nd4j.defaultFloatingPointType());
return rand(dataType, order, ArrayUtil.toLongArray(shape));
}
/**
@ -2851,7 +2851,7 @@ public class Nd4j {
* @return the random ndarray with the specified shape
*/
public static INDArray rand(@NonNull DataType dataType, char order, @NonNull long... shape) {
INDArray ret = Nd4j.createUninitialized(dataType, shape, order).castTo(Nd4j.defaultFloatingPointType());
INDArray ret = Nd4j.createUninitialized(dataType, shape, order);
return rand(ret);
}
@ -2866,7 +2866,7 @@ public class Nd4j {
* @return the random ndarray with the specified shape
*/
public static INDArray rand(@NonNull DataType dataType, @NonNull int... shape) {
INDArray ret = Nd4j.createUninitialized(dataType, ArrayUtil.toLongArray(shape), Nd4j.order()).castTo(Nd4j.defaultFloatingPointType());
INDArray ret = Nd4j.createUninitialized(dataType, ArrayUtil.toLongArray(shape), Nd4j.order());
return rand(ret);
}
@ -2911,7 +2911,7 @@ public class Nd4j {
* @return the random ndarray with the specified shape
*/
public static INDArray rand(long seed, @NonNull long... shape) {
INDArray ret = createUninitialized(shape, Nd4j.order()).castTo(Nd4j.defaultFloatingPointType());//;INSTANCE.rand(shape, seed);
INDArray ret = createUninitialized(shape, Nd4j.order());//;INSTANCE.rand(shape, seed);
return rand(ret, seed);
}
@ -2920,7 +2920,7 @@ public class Nd4j {
*/
@Deprecated
public static INDArray rand(int[] shape, long seed) {
return rand(seed, ArrayUtil.toLongArray(shape)).castTo(Nd4j.defaultFloatingPointType());
return rand(seed, ArrayUtil.toLongArray(shape));
}
@ -2943,7 +2943,7 @@ public class Nd4j {
*/
@Deprecated
public static INDArray rand(int[] shape, @NonNull org.nd4j.linalg.api.rng.Random rng) {
return rand(rng, ArrayUtil.toLongArray(shape)).castTo(Nd4j.defaultFloatingPointType());
return rand(rng, ArrayUtil.toLongArray(shape));
}
/**
@ -2954,7 +2954,7 @@ public class Nd4j {
* @return the random ndarray with the specified shape
*/
public static INDArray rand(@NonNull org.nd4j.linalg.api.rng.Random rng, @NonNull long... shape) {
INDArray ret = createUninitialized(shape, Nd4j.order()).castTo(Nd4j.defaultFloatingPointType()); //INSTANCE.rand(shape, rng);
INDArray ret = createUninitialized(shape, Nd4j.order()); //INSTANCE.rand(shape, rng);
return rand(ret, rng);
}
@ -2963,7 +2963,7 @@ public class Nd4j {
*/
@Deprecated
public static INDArray rand(int[] shape, @NonNull Distribution dist) {
return rand(dist, ArrayUtil.toLongArray(shape)).castTo(Nd4j.defaultFloatingPointType());
return rand(dist, ArrayUtil.toLongArray(shape));
}
/**

9
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java
View File

@ -75,7 +75,8 @@ public class BinarySerde {
ByteBuffer byteBuffer = buffer.hasArray() ? ByteBuffer.allocateDirect(buffer.array().length).put(buffer.array())
.order(ByteOrder.nativeOrder()) : buffer.order(ByteOrder.nativeOrder());
//bump the byte buffer to the proper position
byteBuffer.position(offset);
Buffer buffer1 = (Buffer) byteBuffer;
buffer1.position(offset);
int rank = byteBuffer.getInt();
if (rank < 0)
throw new IllegalStateException("Found negative integer. Corrupt serialization?");
@ -99,7 +100,8 @@ public class BinarySerde {
DataBuffer buff = Nd4j.createBuffer(slice, type, (int) Shape.length(shapeBuff));
//advance past the data
int position = byteBuffer.position() + (buff.getElementSize() * (int) buff.length());
byteBuffer.position(position);
Buffer buffer2 = (Buffer) byteBuffer;
buffer2.position(position);
//create the final array
//TODO: see how to avoid dup here
INDArray arr = Nd4j.createArrayFromShapeBuffer(buff.dup(), shapeBuff.dup());
@ -116,7 +118,8 @@ public class BinarySerde {
INDArray arr = Nd4j.createArrayFromShapeBuffer(compressedDataBuffer.dup(), shapeBuff.dup());
//advance past the data
int compressLength = (int) compressionDescriptor.getCompressedLength();
byteBuffer.position(byteBuffer.position() + compressLength);
Buffer buffer2 = (Buffer) byteBuffer;
buffer2.position(buffer2.position() + compressLength);
return Pair.of(arr, byteBuffer);
}

3
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml
View File

@ -140,6 +140,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<environmentVariables>
@ -162,7 +163,7 @@
Maximum heap size was set to 6g, as a minimum required value for tests run.
Depending on a build machine, default value is not always enough.
-->
<argLine>-Ddtype=float -Dfile.encoding=UTF-8 -Xmx8g</argLine>
<argLine>-Ddtype=float -Dfile.encoding=UTF-8 -Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
</configuration>
</plugin>
<plugin>

3
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java
View File

@ -526,7 +526,8 @@ public class CpuNDArrayFactory extends BaseNativeNDArrayFactory {
public INDArray toFlattened(char order, Collection<INDArray> matrices) {
Preconditions.checkArgument(matrices.size() > 0, "toFlattened expects > 0 operands");
return Nd4j.exec(new Flatten(order, matrices.toArray(new INDArray[matrices.size()])))[0];
return Nd4j.exec(new Flatten(order, matrices.toArray(new INDArray[matrices.size()])))[0]
.castTo(matrices.iterator().next().dataType());
}
@Override

8
nd4j/nd4j-backends/nd4j-backend-impls/pom.xml
View File

@ -124,6 +124,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<environmentVariables>
@ -139,7 +140,12 @@
Maximum heap size was set to 8g, as a minimum required value for tests run.
Depending on a build machine, default value is not always enough.
-->
<argLine>-Xmx2g</argLine>
<argLine>-Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
<forkedProcessTimeoutInSeconds>240</forkedProcessTimeoutInSeconds>
<forkedProcessExitTimeoutInSeconds>240</forkedProcessExitTimeoutInSeconds>
<parallelTestsTimeoutInSeconds>240</parallelTestsTimeoutInSeconds>
<parallelTestsTimeoutForcedInSeconds>240</parallelTestsTimeoutForcedInSeconds>
</configuration>
</plugin>
<plugin>

11
nd4j/nd4j-backends/nd4j-tests/pom.xml
View File

@ -269,6 +269,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<environmentVariables>
@ -304,7 +305,7 @@
For testing large zoo models, this may not be enough (so comment it out).
-->
<argLine>-Dfile.encoding=UTF-8 </argLine>
<argLine>-Dfile.encoding=UTF-8 -Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
</configuration>
</plugin>
</plugins>
@ -350,6 +351,7 @@
</dependency>
</dependencies>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<environmentVariables>
<LD_LIBRARY_PATH>
${nd4j.basedir}/nd4j-backends/nd4j-backend-impls/nd4j-cuda/target/classes
@ -379,7 +381,12 @@
Maximum heap size was set to 6g, as a minimum required value for tests run.
Depending on a build machine, default value is not always enough.
-->
<argLine> -Dfile.encoding=UTF-8 -Djava.library.path="${nd4j.basedir}/nd4j-backends/nd4j-backend-impls/nd4j-cuda/target/classes"</argLine>
<argLine>-Dfile.encoding=UTF-8 -Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
<forkedProcessTimeoutInSeconds>240</forkedProcessTimeoutInSeconds>
<forkedProcessExitTimeoutInSeconds>240</forkedProcessExitTimeoutInSeconds>
<parallelTestsTimeoutInSeconds>240</parallelTestsTimeoutInSeconds>
<parallelTestsTimeoutForcedInSeconds>240</parallelTestsTimeoutForcedInSeconds>
</configuration>
</plugin>
</plugins>

2
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/TestSessions.java
View File

@ -216,6 +216,8 @@ public class TestSessions extends BaseNd4jTestWithBackends {
@Tag(TagNames.FILE_IO)
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testSwitchWhile(Nd4jBackend backend) throws Exception{
/*

13
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/samediff/SameDiffTests.java
View File

@ -94,7 +94,6 @@ import org.nd4j.weightinit.impl.UniformInitScheme;
@Tag(TagNames.SAMEDIFF)
public class SameDiffTests extends BaseNd4jTestWithBackends {
private DataType initialType;
@Override
@ -112,16 +111,11 @@ public class SameDiffTests extends BaseNd4jTestWithBackends {
@BeforeEach
public void before() {
Nd4j.create(1);
initialType = Nd4j.dataType();
Nd4j.setDataType(DataType.DOUBLE);
Nd4j.getRandom().setSeed(123);
}
@AfterEach
public void after() {
Nd4j.setDataType(initialType);
NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false);
NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false);
}
@ -136,7 +130,7 @@ public class SameDiffTests extends BaseNd4jTestWithBackends {
INDArray labels = Nd4j.create(new double[]{1, 1, 0, 1}).reshape(4, 1);
INDArray weights = Nd4j.zeros(3, 1);
INDArray weights = Nd4j.zeros(3, 1).castTo(labels.dataType());
Map<String, INDArray> inputMap = new HashMap<>();
inputMap.put("x", inputs);
@ -155,7 +149,7 @@ public class SameDiffTests extends BaseNd4jTestWithBackends {
val nodeA = sd.math().square(input);
val nodeB = sd.math().square(nodeA);
sd.associateArrayWithVariable(Nd4j.create(new double[]{1, 2, 3, 4, 5, 6}, new long[]{2, 3}), input);
sd.associateArrayWithVariable(Nd4j.create(new double[]{1, 2, 3, 4, 5, 6}, new long[]{2, 3}).castTo(input.dataType()), input);
sd.outputAll(null);
@ -2627,7 +2621,8 @@ public class SameDiffTests extends BaseNd4jTestWithBackends {
SameDiff sd = SameDiff.create();
SDVariable in = sd.placeHolder("in", DataType.FLOAT, 1, 3);
SDVariable w = sd.constant("w", Nd4j.rand(DataType.FLOAT, 3, 4));
INDArray const1 = Nd4j.rand(DataType.FLOAT, 3, 4);
SDVariable w = sd.constant("w",const1);
SDVariable b = sd.var("b", Nd4j.rand(DataType.FLOAT, 1, 4));
SDVariable mmul = in.mmul(w);
SDVariable add = mmul.add(b);

19
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/samediff/listeners/CheckpointListenerTest.java
View File

@ -21,13 +21,18 @@
package org.nd4j.autodiff.samediff.listeners;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.nd4j.autodiff.listeners.checkpoint.CheckpointListener;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.autodiff.samediff.TrainingConfig;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.BaseNd4jTestWithBackends;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.dataset.IrisDataSetIterator;
@ -38,10 +43,7 @@ import org.nd4j.linalg.learning.config.Adam;
import java.io.File;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -169,8 +171,12 @@ public class CheckpointListenerTest extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Execution(ExecutionMode.SAME_THREAD)
@Disabled("Inconsistent results on output")
@Tag(TagNames.NEEDS_VERIFY)
public void testCheckpointListenerEveryTimeUnit(Nd4jBackend backend) throws Exception {
File dir = testDir.toFile();
File dir = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile();
assertTrue(dir.mkdirs());
SameDiff sd = getModel();
CheckpointListener l = new CheckpointListener.Builder(dir)
@ -181,9 +187,8 @@ public class CheckpointListenerTest extends BaseNd4jTestWithBackends {
DataSetIterator iter = getIter(15, 150);
for(int i=0; i<5; i++ ){ //10 iterations total
for(int i = 0; i < 5; i++) { //10 iterations total
sd.fit(iter, 1);
Thread.sleep(5000);
}
//Expect models saved at iterations: 10, 20, 30, 40

2
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/samediff/listeners/ListenerTest.java
View File

@ -123,7 +123,7 @@ public class ListenerTest extends BaseNd4jTestWithBackends {
//
// sd.evaluateMultiple(iter, evalMap);
e = (Evaluation) hist.finalTrainingEvaluations().evaluation(predictions);
e = hist.finalTrainingEvaluations().evaluation(predictions);
System.out.println(e.stats());

10
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/evaluation/RegressionEvalTest.java
View File

@ -79,7 +79,7 @@ public class RegressionEvalTest extends BaseNd4jTestWithBackends {
RegressionEvaluation eval = new RegressionEvaluation(nCols);
for (int i = 0; i < nTestArrays; i++) {
INDArray rand = Nd4j.rand(valuesPerTestArray, nCols).castTo(DataType.DOUBLE);
INDArray rand = Nd4j.rand(DataType.DOUBLE,valuesPerTestArray, nCols);
eval.eval(rand, rand);
}
@ -172,8 +172,8 @@ public class RegressionEvalTest extends BaseNd4jTestWithBackends {
for (int i = 0; i < nEvalInstances; i++) {
list.add(new RegressionEvaluation(nCols));
for (int j = 0; j < numMinibatches; j++) {
INDArray p = Nd4j.rand(nRows, nCols).castTo(Nd4j.defaultFloatingPointType());
INDArray act = Nd4j.rand(nRows, nCols).castTo(Nd4j.defaultFloatingPointType());
INDArray p = Nd4j.rand(DataType.DOUBLE,nRows, nCols);
INDArray act = Nd4j.rand(DataType.DOUBLE,nRows, nCols);
single.eval(act, p);
@ -383,7 +383,7 @@ public class RegressionEvalTest extends BaseNd4jTestWithBackends {
List<INDArray> rowsL = new ArrayList<>();
//Check per-example masking:
INDArray mask1dPerEx = Nd4j.createFromArray(1, 0);
INDArray mask1dPerEx = Nd4j.createFromArray(1, 0).castTo(DataType.FLOAT);
NdIndexIterator iter = new NdIndexIterator(2, 10, 10);
while (iter.hasNext()) {
@ -409,7 +409,7 @@ public class RegressionEvalTest extends BaseNd4jTestWithBackends {
}
//Check per-output masking:
INDArray perOutMask = Nd4j.randomBernoulli(0.5, label.shape());
INDArray perOutMask = Nd4j.randomBernoulli(0.5, label.shape()).castTo(DataType.FLOAT);
rowsP.clear();
rowsL.clear();
List<INDArray> rowsM = new ArrayList<>();

6
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/AveragingTests.java
View File

@ -24,6 +24,8 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -45,18 +47,15 @@ public class AveragingTests extends BaseNd4jTestWithBackends {
private final int THREADS = 16;
private final int LENGTH = 51200 * 4;
DataType initialType = Nd4j.dataType();
@BeforeEach
public void setUp() {
DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
}
@AfterEach
public void shutUp() {
DataTypeUtil.setDTypeForContext(initialType);
}
@ -111,6 +110,7 @@ public class AveragingTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Execution(ExecutionMode.SAME_THREAD)
public void testSingleDeviceAveraging2(Nd4jBackend backend) {
INDArray exp = Nd4j.linspace(1, LENGTH, LENGTH);
List<INDArray> arrays = new ArrayList<>();

4
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/LoneTest.java
View File

@ -23,11 +23,13 @@ package org.nd4j.linalg;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.apache.commons.lang3.RandomUtils;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.nd4j.common.tests.tags.NativeTag;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.transforms.custom.SoftMax;
@ -239,6 +241,8 @@ public class LoneTest extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public void testGetRow1(Nd4jBackend backend) {
INDArray array = Nd4j.create(10000, 10000);

39
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java
View File

@ -29,6 +29,9 @@ import org.apache.commons.math3.util.FastMath;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.api.parallel.Isolated;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -148,8 +151,6 @@ import static org.junit.jupiter.api.Assertions.*;
@Tag(TagNames.FILE_IO)
public class Nd4jTestsC extends BaseNd4jTestWithBackends {
DataType initialType = Nd4j.dataType();
Level1 l1 = Nd4j.getBlasWrapper().level1();
@TempDir Path testDir;
@Override
@ -159,7 +160,6 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@BeforeEach
public void before() throws Exception {
Nd4j.setDataType(DataType.DOUBLE);
Nd4j.getRandom().setSeed(123);
Nd4j.getExecutioner().enableDebugMode(false);
Nd4j.getExecutioner().enableVerboseMode(false);
@ -167,7 +167,6 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@AfterEach
public void after() throws Exception {
Nd4j.setDataType(initialType);
}
@ParameterizedTest
@ -1480,7 +1479,7 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
}
INDArray assertion = Nd4j.create(new double[] {1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4}, new int[]{12});
INDArray flattened = Nd4j.toFlattened(concat);
INDArray flattened = Nd4j.toFlattened(concat).castTo(assertion.dataType());
assertEquals(assertion, flattened);
}
@ -3902,6 +3901,8 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Crashes")
@Tag(TagNames.NEEDS_VERIFY)
public void testSingleDeviceAveraging(Nd4jBackend backend) {
int LENGTH = 512 * 1024 * 2;
INDArray array1 = Nd4j.valueArrayOf(LENGTH, 1.0);
@ -5587,6 +5588,8 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Crashes")
@Tag(TagNames.NEEDS_VERIFY)
public void testNativeSort3(Nd4jBackend backend) {
int length = isIntegrationTests() ? 1048576 : 16484;
INDArray array = Nd4j.linspace(1, length, length, DataType.DOUBLE).reshape(1, -1);
@ -5719,6 +5722,8 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Crashes")
@Tag(TagNames.NEEDS_VERIFY)
public void testNativeSortAlongDimension1(Nd4jBackend backend) {
INDArray array = Nd4j.create(1000, 1000);
INDArray exp1 = Nd4j.linspace(1, 1000, 1000, DataType.DOUBLE);
@ -5779,6 +5784,8 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Crashes")
@Tag(TagNames.NEEDS_VERIFY)
public void testNativeSortAlongDimension3(Nd4jBackend backend) {
INDArray array = Nd4j.create(2000, 2000);
INDArray exp1 = Nd4j.linspace(1, 2000, 2000, DataType.DOUBLE);
@ -5814,6 +5821,8 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Crashes")
@Tag(TagNames.NEEDS_VERIFY)
public void testNativeSortAlongDimension2(Nd4jBackend backend) {
INDArray array = Nd4j.create(100, 10);
INDArray exp1 = Nd4j.create(new double[] {9, 8, 7, 6, 5, 4, 3, 2, 1, 0});
@ -6768,15 +6777,16 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testInconsistentOutput(){
@Execution(ExecutionMode.SAME_THREAD)
public void testInconsistentOutput(Nd4jBackend backend) {
INDArray in = Nd4j.rand(1, 802816).castTo(DataType.DOUBLE);
INDArray W = Nd4j.rand(802816, 1).castTo(DataType.DOUBLE);
INDArray b = Nd4j.create(1).castTo(DataType.DOUBLE);
INDArray out = fwd(in, W, b);
for(int i = 0;i < 100;i++) {
for(int i = 0; i < 100;i++) {
INDArray out2 = fwd(in, W, b); //l.activate(inToLayer1, false, LayerWorkspaceMgr.noWorkspaces());
assertEquals( out, out2,"Failed at iteration [" + String.valueOf(i) + "]");
assertEquals( out, out2,"Failed at iteration [" + i + "]");
}
}
@ -7144,9 +7154,10 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testRowColumnOpsRank1(){
@Execution(ExecutionMode.SAME_THREAD)
public void testRowColumnOpsRank1(Nd4jBackend backend) {
for( int i=0; i<6; i++ ) {
for( int i = 0; i < 6; i++ ) {
INDArray orig = Nd4j.linspace(1, 12, 12, DataType.DOUBLE).reshape('c', 3, 4);
INDArray in1r = orig.dup();
INDArray in2r = orig.dup();
@ -7954,6 +7965,8 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Crashes")
@Tag(TagNames.NEEDS_VERIFY)
public void testRollingMean(Nd4jBackend backend) {
val wsconf = WorkspaceConfiguration.builder()
.initialSize(4L * (32*128*256*256 + 32*128 + 10*1024*1024))
@ -8558,8 +8571,6 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled("Needs verification")
@Tag(TagNames.NEEDS_VERIFY)
public void testBatchToSpace(Nd4jBackend backend) {
INDArray out = Nd4j.create(DataType.FLOAT, 2, 4, 5);
@ -8833,7 +8844,9 @@ public class Nd4jTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testCreateBufferFromByteBuffer(){
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public void testCreateBufferFromByteBuffer(Nd4jBackend backend){
for(DataType dt : DataType.values()){
if(dt == DataType.COMPRESSED || dt == DataType.UTF8 || dt == DataType.UNKNOWN)

4
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/api/buffer/DataBufferTests.java
View File

@ -41,6 +41,7 @@ import org.nd4j.linalg.factory.Nd4jBackend;
import org.nd4j.nativeblas.NativeOpsHolder;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@ -378,7 +379,8 @@ public class DataBufferTests extends BaseNd4jTestWithBackends {
INDArray arr2 = Nd4j.create(dt, arr.shape());
ByteBuffer bb = arr2.data().pointer().asByteBuffer();
bb.position(0);
Buffer buffer = (Buffer) bb;
buffer.position(0);
bb.put(b);
Nd4j.getAffinityManager().tagLocation(arr2, AffinityManager.Location.HOST);

17
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/api/buffer/FloatDataBufferTest.java
View File

@ -59,18 +59,15 @@ import static org.junit.jupiter.api.Assertions.*;
@NativeTag
public class FloatDataBufferTest extends BaseNd4jTestWithBackends {
DataType initialType = Nd4j.dataType();
@TempDir Path tempDir;
@BeforeEach
public void before() {
DataTypeUtil.setDTypeForContext(DataType.FLOAT);
System.out.println("DATATYPE HERE: " + Nd4j.dataType());
}
@AfterEach
public void after() {
DataTypeUtil.setDTypeForContext(initialType);
}
@ -191,7 +188,7 @@ public class FloatDataBufferTest extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testAsBytes(Nd4jBackend backend) {
INDArray arr = Nd4j.create(5);
INDArray arr = Nd4j.create(DataType.FLOAT,5);
byte[] d = arr.data().asBytes();
assertEquals(4 * 5, d.length,getFailureMessage(backend));
INDArray rand = Nd4j.rand(3, 3);
@ -245,7 +242,9 @@ public class FloatDataBufferTest extends BaseNd4jTestWithBackends {
buffer.reallocate(6);
float[] newBuf = buffer.asFloat();
assertEquals(6, buffer.capacity());
assertArrayEquals(old, newBuf, 1e-4F);
//note: old and new buf are not equal because java automatically populates the arrays with zeros
//the new buffer is actually 1,2,3,4,0,0 because of this
assertArrayEquals(new float[]{1,2,3,4,0,0}, newBuf, 1e-4F);
}
@ParameterizedTest
@ -253,8 +252,7 @@ public class FloatDataBufferTest extends BaseNd4jTestWithBackends {
public void testReallocationWorkspace(Nd4jBackend backend) {
WorkspaceConfiguration initialConfig = WorkspaceConfiguration.builder().initialSize(10 * 1024L * 1024L)
.policyAllocation(AllocationPolicy.STRICT).policyLearning(LearningPolicy.NONE).build();
MemoryWorkspace workspace = Nd4j.getWorkspaceManager().getAndActivateWorkspace(initialConfig, "SOME_ID");
try(MemoryWorkspace workspace = Nd4j.getWorkspaceManager().getAndActivateWorkspace(initialConfig, "SOME_ID")) {
DataBuffer buffer = Nd4j.createBuffer(new float[] {1, 2, 3, 4});
assertTrue(buffer.isAttached());
float[] old = buffer.asFloat();
@ -262,8 +260,9 @@ public class FloatDataBufferTest extends BaseNd4jTestWithBackends {
buffer.reallocate(6);
assertEquals(6, buffer.capacity());
float[] newBuf = buffer.asFloat();
assertArrayEquals(old, newBuf, 1e-4F);
workspace.close();
//note: java creates new zeros by default for empty array spots
assertArrayEquals(new float[]{1,2,3,4,0,0}, newBuf, 1e-4F);
}
}
@ParameterizedTest

2
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/broadcast/BasicBroadcastTests.java
View File

@ -175,9 +175,11 @@ public class BasicBroadcastTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void basicBroadcastFailureTest_4(Nd4jBackend backend) {
assertThrows(IllegalStateException.class,() -> {
val x = Nd4j.create(DataType.FLOAT, 3, 1, 2).assign(4.f);
val y = Nd4j.createFromArray(new float[]{2.f, 2.f, 2.f, 2.f}).reshape(2, 2);
val z = x.addi(y);
});
}
@ParameterizedTest

11
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/compression/CompressionTests.java
View File

@ -25,6 +25,9 @@ import lombok.val;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.api.parallel.Isolated;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -52,6 +55,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Slf4j
@NativeTag
@Tag(TagNames.COMPRESSION)
@Isolated
@Execution(ExecutionMode.SAME_THREAD)
public class CompressionTests extends BaseNd4jTestWithBackends {
@ -412,9 +417,11 @@ public class CompressionTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testBitmapEncoding2(Nd4jBackend backend) {
INDArray initial = Nd4j.create(40000000);
INDArray target = Nd4j.create(initial.length());
INDArray initial = Nd4j.create(DataType.FLOAT,40000000);
INDArray target = Nd4j.create(DataType.FLOAT,initial.length());
initial.addi(1e-3);

1
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/convolution/DeconvTests.java
View File

@ -61,6 +61,7 @@ public class DeconvTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LARGE_RESOURCES)
public void compareKeras(Nd4jBackend backend) throws Exception {
File newFolder = testDir.toFile();
new ClassPathResource("keras/deconv/").copyDirectory(newFolder);

58
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/custom/CustomOpsTests.java
View File

@ -103,7 +103,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@Override
public char ordering(){
public char ordering() {
return 'c';
}
@ -566,7 +566,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testStridedSliceEdgeCase(){
public void testStridedSliceEdgeCase(Nd4jBackend backend) {
INDArray in = Nd4j.scalar(10.0).reshape(1); //Int [1]
INDArray begin = Nd4j.ones(DataType.INT, 1);
INDArray end = Nd4j.zeros(DataType.INT, 1);
@ -595,7 +595,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testDepthwise(){
public void testDepthwise(Nd4jBackend backend) {
INDArray input = Nd4j.create(DataType.DOUBLE, 1,3,8,8);
INDArray depthwiseWeight = Nd4j.create(DataType.DOUBLE, 1,1,3,2);
INDArray bias = Nd4j.create(DataType.DOUBLE, 1, 6);
@ -660,8 +660,10 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
assertEquals(e, z);
}
@Test()
public void testInputValidationMergeMax(){
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testInputValidationMergeMax(Nd4jBackend backend) {
assertThrows(RuntimeException.class,() -> {
INDArray[] inputs = new INDArray[]{
Nd4j.createFromArray(0.0f, 1.0f, 2.0f).reshape('c', 1, 3),
@ -683,7 +685,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testUpsampling2dBackprop(){
public void testUpsampling2dBackprop(Nd4jBackend backend) {
Nd4j.getRandom().setSeed(12345);
int c = 2;
@ -729,7 +731,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testIsMaxView(){
public void testIsMaxView(Nd4jBackend backend) {
INDArray predictions = Nd4j.rand(DataType.FLOAT, 3, 4, 3, 2);
INDArray row = predictions.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(0), NDArrayIndex.point(0));
@ -748,7 +750,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void isMax4d_2dims(){
public void isMax4d_2dims(Nd4jBackend backend) {
Nd4j.getRandom().setSeed(12345);
INDArray in = Nd4j.rand(DataType.FLOAT, 3, 3, 4, 4).permute(0, 2, 3, 1);
@ -764,7 +766,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testSizeTypes(){
public void testSizeTypes(Nd4jBackend backend) {
List<DataType> failed = new ArrayList<>();
for(DataType dt : new DataType[]{DataType.LONG, DataType.INT, DataType.SHORT, DataType.BYTE,
DataType.UINT64, DataType.UINT32, DataType.UINT16, DataType.UBYTE,
@ -796,7 +798,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testListDiff(){
public void testListDiff(Nd4jBackend backend) {
INDArray x = Nd4j.createFromArray(0, 1, 2, 3);
INDArray y = Nd4j.createFromArray(3, 1);
@ -817,7 +819,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testTopK1(){
public void testTopK1(Nd4jBackend backend) {
INDArray x = Nd4j.createFromArray(0.0, 0.0, 0.0, 10.0, 0.0);
INDArray k = Nd4j.scalar(1);
INDArray outValue = Nd4j.create(DataType.DOUBLE, 1);
@ -897,7 +899,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testAdjustContrastShape(){
public void testAdjustContrastShape(Nd4jBackend backend) {
DynamicCustomOp op = DynamicCustomOp.builder("adjust_contrast_v2")
.addInputs(Nd4j.create(DataType.FLOAT, 256, 256,3), Nd4j.scalar(0.5f))
.build();
@ -910,7 +912,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBitCastShape(){
public void testBitCastShape(Nd4jBackend backend) {
INDArray out = Nd4j.createUninitialized(1,10);
BitCast op = new BitCast(Nd4j.zeros(1,10), DataType.FLOAT.toInt(), out);
List<LongShapeDescriptor> lsd = op.calculateOutputShape();
@ -1148,7 +1150,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testRange(){
public void testRange(Nd4jBackend backend) {
DynamicCustomOp op = DynamicCustomOp.builder("range")
.addFloatingPointArguments(-1.0, 1.0, 0.01)
.build();
@ -1163,7 +1165,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBitCastShape_1(){
public void testBitCastShape_1(Nd4jBackend backend) {
val out = Nd4j.createUninitialized(1,10);
BitCast op = new BitCast(Nd4j.zeros(DataType.FLOAT,1,10), DataType.INT.toInt(), out);
List<LongShapeDescriptor> lsd = op.calculateOutputShape();
@ -1174,7 +1176,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBitCastShape_2(){
public void testBitCastShape_2(Nd4jBackend backend) {
val out = Nd4j.createUninitialized(1,10);
BitCast op = new BitCast(Nd4j.zeros(DataType.DOUBLE,1,10), DataType.INT.toInt(), out);
List<LongShapeDescriptor> lsd = op.calculateOutputShape();
@ -1283,8 +1285,6 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.NEEDS_VERIFY)
@Disabled("Implementation needs verification")
public void testPolygamma(Nd4jBackend backend) {
INDArray n = Nd4j.linspace(DataType.DOUBLE, 1.0, 1.0, 9).reshape(3,3);
INDArray x = Nd4j.create(DataType.DOUBLE, 3,3);
@ -1292,7 +1292,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
INDArray expected = Nd4j.createFromArray(new double[]{4.934802, -16.828796, 97.409088, -771.474243,
7691.113770f, -92203.460938f, 1290440.250000, -20644900.000000, 3.71595e+08}).reshape(3,3);
INDArray output = Nd4j.create(DataType.DOUBLE, expected.shape());
val op = new Polygamma(x,n,output);
val op = new Polygamma(n,x,output);
Nd4j.exec(op);
assertEquals(expected, output);
}
@ -1424,7 +1424,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testAdjustHueShape(){
public void testAdjustHueShape(Nd4jBackend backend) {
INDArray image = Nd4j.createFromArray(new float[]{0.7788f, 0.8012f, 0.7244f,
0.2309f, 0.7271f, 0.1804f, 0.5056f, 0.8925f, 0.5461f,
0.9234f, 0.0856f, 0.7938f, 0.6591f, 0.5555f, 0.1596f,
@ -1470,7 +1470,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBitCastShape_3(){
public void testBitCastShape_3(Nd4jBackend backend) {
val x = Nd4j.createFromArray(new int[]{1, 2, 3, 4, 5, 6, 7, 8}).reshape(1, 4, 2);
val e = Nd4j.createFromArray(new long[]{8589934593L, 17179869187L, 25769803781L, 34359738375L}).reshape(1, 4);
val z = Nd4j.exec(new BitCast(x, DataType.LONG.toInt()))[0];
@ -1958,7 +1958,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBatchNormBpNHWC(){
public void testBatchNormBpNHWC(Nd4jBackend backend) {
//Nd4j.getEnvironment().allowHelpers(false); //Passes if helpers/MKLDNN is disabled
INDArray in = Nd4j.rand(DataType.FLOAT, 2, 4, 4, 3);
@ -1971,13 +1971,13 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
assertEquals(eps, epsStrided);
INDArray out1eps = in.like();
INDArray out1m = mean.like();
INDArray out1v = var.like();
INDArray out1eps = in.like().castTo(DataType.FLOAT);
INDArray out1m = mean.like().castTo(DataType.FLOAT);
INDArray out1v = var.like().castTo(DataType.FLOAT);
INDArray out2eps = in.like();
INDArray out2m = mean.like();
INDArray out2v = var.like();
INDArray out2eps = in.like().castTo(DataType.FLOAT);
INDArray out2m = mean.like().castTo(DataType.FLOAT);
INDArray out2v = var.like().castTo(DataType.FLOAT);
DynamicCustomOp op1 = DynamicCustomOp.builder("batchnorm_bp")
.addInputs(in, mean, var, gamma, beta, eps)
@ -2004,7 +2004,7 @@ public class CustomOpsTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testSpaceToDepthBadStrides(){
public void testSpaceToDepthBadStrides(Nd4jBackend backend) {
INDArray in = Nd4j.rand(DataType.FLOAT, 2, 3, 6, 6);
INDArray inBadStrides = in.permute(1,0,2,3).dup().permute(1,0,2,3);
assertEquals(in, inBadStrides);

126
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/dataset/NormalizerSerializerTest.java
View File

@ -24,6 +24,7 @@ import lombok.Getter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -45,10 +46,13 @@ import org.nd4j.linalg.dataset.api.preprocessor.stats.DistributionStats;
import org.nd4j.linalg.dataset.api.preprocessor.stats.MinMaxStats;
import org.nd4j.linalg.dataset.api.preprocessor.stats.NormalizerStats;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.factory.Nd4jBackend;
import java.io.*;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import static java.util.Arrays.asList;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -61,83 +65,91 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
@NativeTag
@Tag(TagNames.FILE_IO)
public class NormalizerSerializerTest extends BaseNd4jTestWithBackends {
private File tmpFile;
@TempDir File tmpFile;
private NormalizerSerializer SUT;
@BeforeEach
public void setUp() throws IOException {
tmpFile = File.createTempFile("test", "preProcessor");
tmpFile.deleteOnExit();
SUT = NormalizerSerializer.getDefault();
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testImagePreProcessingScaler() throws Exception {
public void testImagePreProcessingScaler(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
ImagePreProcessingScaler imagePreProcessingScaler = new ImagePreProcessingScaler(0,1);
SUT.write(imagePreProcessingScaler,tmpFile);
SUT.write(imagePreProcessingScaler,normalizerFile);
ImagePreProcessingScaler restored = SUT.restore(tmpFile);
ImagePreProcessingScaler restored = SUT.restore(normalizerFile);
assertEquals(imagePreProcessingScaler,restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNormalizerStandardizeNotFitLabels() throws Exception {
public void testNormalizerStandardizeNotFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
NormalizerStandardize original = new NormalizerStandardize(Nd4j.create(new double[] {0.5, 1.5}).reshape(1, -1),
Nd4j.create(new double[] {2.5, 3.5}).reshape(1, -1));
SUT.write(original, tmpFile);
NormalizerStandardize restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
NormalizerStandardize restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNormalizerStandardizeFitLabels() throws Exception {
public void testNormalizerStandardizeFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
NormalizerStandardize original = new NormalizerStandardize(Nd4j.create(new double[] {0.5, 1.5}).reshape(1, -1),
Nd4j.create(new double[] {2.5, 3.5}).reshape(1, -1), Nd4j.create(new double[] {4.5, 5.5}).reshape(1, -1),
Nd4j.create(new double[] {6.5, 7.5}).reshape(1, -1));
original.fitLabel(true);
SUT.write(original, tmpFile);
NormalizerStandardize restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
NormalizerStandardize restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNormalizerMinMaxScalerNotFitLabels() throws Exception {
public void testNormalizerMinMaxScalerNotFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
NormalizerMinMaxScaler original = new NormalizerMinMaxScaler(0.1, 0.9);
original.setFeatureStats(Nd4j.create(new double[] {0.5, 1.5}).reshape(1, -1), Nd4j.create(new double[] {2.5, 3.5}).reshape(1, -1));
SUT.write(original, tmpFile);
NormalizerMinMaxScaler restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
NormalizerMinMaxScaler restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNormalizerMinMaxScalerFitLabels() throws Exception {
public void testNormalizerMinMaxScalerFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
NormalizerMinMaxScaler original = new NormalizerMinMaxScaler(0.1, 0.9);
original.setFeatureStats(Nd4j.create(new double[] {0.5, 1.5}), Nd4j.create(new double[] {2.5, 3.5}));
original.setLabelStats(Nd4j.create(new double[] {4.5, 5.5}), Nd4j.create(new double[] {6.5, 7.5}));
original.fitLabel(true);
SUT.write(original, tmpFile);
NormalizerMinMaxScaler restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
NormalizerMinMaxScaler restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerStandardizeNotFitLabels() throws Exception {
public void testMultiNormalizerStandardizeNotFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerStandardize original = new MultiNormalizerStandardize();
original.setFeatureStats(asList(
new DistributionStats(Nd4j.create(new double[] {0.5, 1.5}).reshape(1, -1),
@ -145,15 +157,17 @@ public class NormalizerSerializerTest extends BaseNd4jTestWithBackends {
new DistributionStats(Nd4j.create(new double[] {4.5, 5.5, 6.5}).reshape(1, -1),
Nd4j.create(new double[] {7.5, 8.5, 9.5}).reshape(1, -1))));
SUT.write(original, tmpFile);
MultiNormalizerStandardize restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerStandardize restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerStandardizeFitLabels() throws Exception {
public void testMultiNormalizerStandardizeFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerStandardize original = new MultiNormalizerStandardize();
original.setFeatureStats(asList(
new DistributionStats(Nd4j.create(new double[] {0.5, 1.5}).reshape(1, -1),
@ -168,30 +182,34 @@ public class NormalizerSerializerTest extends BaseNd4jTestWithBackends {
Nd4j.create(new double[] {7.5, 8.5, 9.5}).reshape(1, -1))));
original.fitLabel(true);
SUT.write(original, tmpFile);
MultiNormalizerStandardize restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerStandardize restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerMinMaxScalerNotFitLabels() throws Exception {
public void testMultiNormalizerMinMaxScalerNotFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerMinMaxScaler original = new MultiNormalizerMinMaxScaler(0.1, 0.9);
original.setFeatureStats(asList(
new MinMaxStats(Nd4j.create(new double[] {0.5, 1.5}), Nd4j.create(new double[] {2.5, 3.5})),
new MinMaxStats(Nd4j.create(new double[] {4.5, 5.5, 6.5}),
Nd4j.create(new double[] {7.5, 8.5, 9.5}))));
SUT.write(original, tmpFile);
MultiNormalizerMinMaxScaler restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerMinMaxScaler restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerMinMaxScalerFitLabels() throws Exception {
public void testMultiNormalizerMinMaxScalerFitLabels(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerMinMaxScaler original = new MultiNormalizerMinMaxScaler(0.1, 0.9);
original.setFeatureStats(asList(
new MinMaxStats(Nd4j.create(new double[] {0.5, 1.5}), Nd4j.create(new double[] {2.5, 3.5})),
@ -204,28 +222,32 @@ public class NormalizerSerializerTest extends BaseNd4jTestWithBackends {
Nd4j.create(new double[] {7.5, 8.5, 9.5}))));
original.fitLabel(true);
SUT.write(original, tmpFile);
MultiNormalizerMinMaxScaler restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerMinMaxScaler restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerHybridEmpty() throws Exception {
public void testMultiNormalizerHybridEmpty(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerHybrid original = new MultiNormalizerHybrid();
original.setInputStats(new HashMap<Integer, NormalizerStats>());
original.setOutputStats(new HashMap<Integer, NormalizerStats>());
original.setInputStats(new HashMap<>());
original.setOutputStats(new HashMap<>());
SUT.write(original, tmpFile);
MultiNormalizerHybrid restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerHybrid restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerHybridGlobalStats() throws Exception {
public void testMultiNormalizerHybridGlobalStats(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerHybrid original = new MultiNormalizerHybrid().minMaxScaleAllInputs().standardizeAllOutputs();
Map<Integer, NormalizerStats> inputStats = new HashMap<>();
@ -239,15 +261,17 @@ public class NormalizerSerializerTest extends BaseNd4jTestWithBackends {
original.setInputStats(inputStats);
original.setOutputStats(outputStats);
SUT.write(original, tmpFile);
MultiNormalizerHybrid restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerHybrid restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultiNormalizerHybridGlobalAndSpecificStats() throws Exception {
public void testMultiNormalizerHybridGlobalAndSpecificStats(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MultiNormalizerHybrid original = new MultiNormalizerHybrid().standardizeAllInputs().minMaxScaleInput(0, -5, 5)
.minMaxScaleAllOutputs(-10, 10).standardizeOutput(1);
@ -262,29 +286,35 @@ public class NormalizerSerializerTest extends BaseNd4jTestWithBackends {
original.setInputStats(inputStats);
original.setOutputStats(outputStats);
SUT.write(original, tmpFile);
MultiNormalizerHybrid restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MultiNormalizerHybrid restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}
@Test()
public void testCustomNormalizerWithoutRegisteredStrategy() throws Exception {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testCustomNormalizerWithoutRegisteredStrategy(Nd4jBackend backend) throws Exception {
assertThrows(RuntimeException.class, () -> {
SUT.write(new MyNormalizer(123), tmpFile);
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
SUT.write(new MyNormalizer(123), normalizerFile);
});
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testCustomNormalizer() throws Exception {
public void testCustomNormalizer(Nd4jBackend backend) throws Exception {
File normalizerFile = Files.createTempFile(tmpFile.toPath(),"pre-process-" + UUID.randomUUID().toString(),"bin").toFile();
MyNormalizer original = new MyNormalizer(42);
SUT.addStrategy(new MyNormalizerSerializerStrategy());
SUT.write(original, tmpFile);
MyNormalizer restored = SUT.restore(tmpFile);
SUT.write(original, normalizerFile);
MyNormalizer restored = SUT.restore(normalizerFile);
assertEquals(original, restored);
}

4
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/factory/Nd4jTest.java
View File

@ -42,6 +42,7 @@ import org.nd4j.common.util.ArrayUtil;
import org.nd4j.nativeblas.NativeOpsHolder;
import java.io.File;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@ -248,7 +249,8 @@ public class Nd4jTest extends BaseNd4jTestWithBackends {
byte[] dataTwo = new byte[floatBuffer.capacity()];
floatBuffer.get(dataTwo);
assertArrayEquals(originalData,dataTwo);
floatBuffer.position(0);
Buffer buffer = (Buffer) floatBuffer;
buffer.position(0);
DataBuffer dataBuffer = Nd4j.createBuffer(new FloatPointer(floatBuffer.asFloatBuffer()),linspace.length(), DataType.FLOAT);
assertArrayEquals(new float[]{1,2,3,4}, dataBuffer.asFloat(), 1e-5f);

10
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/ops/OpExecutionerTests.java
View File

@ -23,6 +23,8 @@ package org.nd4j.linalg.ops;
import lombok.val;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -116,8 +118,10 @@ public class OpExecutionerTests extends BaseNd4jTestWithBackends {
}
@Test
public void testDistance() throws Exception {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Execution(ExecutionMode.SAME_THREAD)
public void testDistance(Nd4jBackend backend) throws Exception {
INDArray matrix = Nd4j.rand(new int[] {400,10});
INDArray rowVector = matrix.getRow(70);
INDArray resultArr = Nd4j.zeros(400,1);
@ -127,8 +131,6 @@ public class OpExecutionerTests extends BaseNd4jTestWithBackends {
System.out.println("Ran!");
});
Thread.sleep(600000);
}
@ParameterizedTest

2
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/ops/OpExecutionerTestsC.java
View File

@ -82,11 +82,9 @@ import static org.nd4j.linalg.indexing.NDArrayIndex.point;
@Slf4j
@NativeTag
public class OpExecutionerTestsC extends BaseNd4jTestWithBackends {
DataType initialType = Nd4j.dataType();
@AfterEach
public void after() {
Nd4j.setDataType(this.initialType);
}

3
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/profiling/InfNanTests.java
View File

@ -23,6 +23,8 @@ package org.nd4j.linalg.profiling;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -39,6 +41,7 @@ import org.nd4j.linalg.profiler.ProfilerConfig;
import static org.junit.jupiter.api.Assertions.assertThrows;
@NativeTag
@Execution(ExecutionMode.SAME_THREAD)
public class InfNanTests extends BaseNd4jTestWithBackends {

12
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/profiling/OperationProfilerTests.java
View File

@ -27,6 +27,9 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.api.parallel.Isolated;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.nd4j.common.tests.tags.NativeTag;
@ -52,6 +55,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Slf4j
@NativeTag
@Isolated
@Execution(ExecutionMode.SAME_THREAD)
public class OperationProfilerTests extends BaseNd4jTestWithBackends {
@ -229,9 +234,10 @@ public class OperationProfilerTests extends BaseNd4jTestWithBackends {
assertTrue(ArrayUtils.contains(causes, OpProfiler.PenaltyCause.TAD_NON_EWS_ACCESS));
}
@Test
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testBadTad4(Nd4jBackend backend) {
INDArray x = Nd4j.create(2, 4, 5, 6);
INDArray x = Nd4j.create(DataType.DOUBLE,2, 4, 5, 6);
Pair<DataBuffer, DataBuffer> pair = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(x, 3);
@ -473,7 +479,7 @@ public class OperationProfilerTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNanPanic(){
public void testNanPanic(Nd4jBackend backend) {
try {
DynamicCustomOp op = DynamicCustomOp.builder("add")
.addInputs(Nd4j.valueArrayOf(10, Double.NaN).castTo(DataType.DOUBLE), Nd4j.scalar(0.0))

11
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/rng/RandomTests.java
View File

@ -441,6 +441,7 @@ public class RandomTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testStepOver1(Nd4jBackend backend) {
Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
@ -466,6 +467,8 @@ public class RandomTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testSum_119(Nd4jBackend backend) {
INDArray z2 = Nd4j.zeros(DataType.DOUBLE, 55000000);
val sum = z2.sumNumber().doubleValue();
@ -474,6 +477,8 @@ public class RandomTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testLegacyDistribution1(Nd4jBackend backend) {
NormalDistribution distribution = new NormalDistribution(new DefaultRandom(), 0.0, 1.0);
INDArray z1 = distribution.sample(new int[] {1, 1000000});
@ -923,9 +928,10 @@ public class RandomTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testDeallocation1() throws Exception {
while (true) {
for(int i = 0; i < 1000; i++) {
Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
random1.nextInt();
@ -934,6 +940,7 @@ public class RandomTests extends BaseNd4jTestWithBackends {
}
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void someTest(Nd4jBackend backend) {

9
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/rng/RngValidationTests.java
View File

@ -29,6 +29,8 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.nd4j.common.base.Preconditions;
@ -70,6 +72,7 @@ import java.util.Map;
@Slf4j
@NativeTag
@Tag(TagNames.RNG)
@Execution(ExecutionMode.SAME_THREAD)
public class RngValidationTests extends BaseNd4jTestWithBackends {
@ -129,6 +132,8 @@ public class RngValidationTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled
@Tag(TagNames.NEEDS_VERIFY)
public void validateRngDistributions(Nd4jBackend backend){
List<TestCase> testCases = new ArrayList<>();
for(DataType type : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
@ -264,7 +269,7 @@ public class RngValidationTests extends BaseNd4jTestWithBackends {
int count = 1;
for(TestCase tc : testCases){
for(TestCase tc : testCases) {
log.info("Starting test case: {} of {}", count, testCases.size());
log.info("{}", tc);
@ -314,7 +319,7 @@ public class RngValidationTests extends BaseNd4jTestWithBackends {
assertEquals(z, z2);
//Check mean, stdev
if(tc.getExpectedMean() != null){
if(tc.getExpectedMean() != null) {
double mean = z.meanNumber().doubleValue();
double re = relError(tc.getExpectedMean(), mean);
double ae = Math.abs(tc.getExpectedMean() - mean);

2
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/serde/LargeSerDeTests.java
View File

@ -44,6 +44,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
@Slf4j
@Tag(TagNames.JACKSON_SERDE)
@NativeTag
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public class LargeSerDeTests extends BaseNd4jTestWithBackends {
@ParameterizedTest

15
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/serde/NumpyFormatTests.java
View File

@ -42,6 +42,7 @@ import org.nd4j.common.io.ClassPathResource;
import java.io.File;
import java.nio.file.Path;
import java.util.Map;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.*;
@ -56,7 +57,8 @@ public class NumpyFormatTests extends BaseNd4jTestWithBackends {
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testToNpyFormat(Nd4jBackend backend) throws Exception {
val dir = testDir.toFile();
val dir = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile();
assertTrue(dir.mkdirs());
new ClassPathResource("numpy_arrays/").copyDirectory(dir);
File[] files = dir.listFiles();
@ -107,14 +109,15 @@ public class NumpyFormatTests extends BaseNd4jTestWithBackends {
public void testToNpyFormatScalars(Nd4jBackend backend) throws Exception {
// File dir = new File("C:\\DL4J\\Git\\dl4j-test-resources\\src\\main\\resources\\numpy_arrays\\scalar");
val dir = testDir.toFile();
val dir = testDir.resolve("new-path0" + UUID.randomUUID().toString()).toFile();
dir.mkdirs();
new ClassPathResource("numpy_arrays/scalar/").copyDirectory(dir);
File[] files = dir.listFiles();
int cnt = 0;
for(File f : files){
if(!f.getPath().endsWith(".npy")){
if(!f.getPath().endsWith(".npy")) {
log.warn("Skipping: {}", f);
continue;
}
@ -161,7 +164,8 @@ public class NumpyFormatTests extends BaseNd4jTestWithBackends {
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNpzReading(Nd4jBackend backend) throws Exception {
val dir = testDir.toFile();
val dir = testDir.resolve("new-folder-npz").toFile();
dir.mkdirs();
new ClassPathResource("numpy_arrays/npz/").copyDirectory(dir);
File[] files = dir.listFiles();
@ -222,7 +226,8 @@ public class NumpyFormatTests extends BaseNd4jTestWithBackends {
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testNpy(Nd4jBackend backend) throws Exception {
for(boolean empty : new boolean[]{false, true}) {
val dir = testDir.toFile();
val dir = testDir.resolve("new-dir-1-" + UUID.randomUUID().toString()).toFile();
assertTrue(dir.mkdirs());
if(!empty) {
new ClassPathResource("numpy_arrays/npy/3,4/").copyDirectory(dir);
} else {

8
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/shape/ShapeTestsC.java
View File

@ -403,13 +403,13 @@ public class ShapeTestsC extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testRavel(Nd4jBackend backend) {
INDArray linspace = Nd4j.linspace(1, 4, 4).reshape(2, 2);
INDArray asseriton = Nd4j.linspace(1, 4, 4);
INDArray linspace = Nd4j.linspace(1, 4, 4,DataType.DOUBLE).reshape(2, 2);
INDArray asseriton = Nd4j.linspace(1, 4, 4,DataType.DOUBLE);
INDArray raveled = linspace.ravel();
assertEquals(asseriton, raveled);
INDArray tensorLinSpace = Nd4j.linspace(1, 16, 16).reshape(2, 2, 2, 2);
INDArray linspaced = Nd4j.linspace(1, 16, 16);
INDArray tensorLinSpace = Nd4j.linspace(1, 16, 16,DataType.DOUBLE).reshape(2, 2, 2, 2);
INDArray linspaced = Nd4j.linspace(1, 16, 16,DataType.DOUBLE);
INDArray tensorLinspaceRaveled = tensorLinSpace.ravel();
assertEquals(linspaced, tensorLinspaceRaveled);

1
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/shape/concat/ConcatTestsC.java
View File

@ -236,7 +236,6 @@ public class ConcatTestsC extends BaseNd4jTestWithBackends {
});
}
@Test
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testConcat3dv2(Nd4jBackend backend) {

48
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/specials/LongTests.java
View File

@ -21,9 +21,9 @@
package org.nd4j.linalg.specials;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.api.parallel.Isolated;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -49,19 +49,30 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals;
@Slf4j
@NativeTag
@Isolated
@Execution(ExecutionMode.SAME_THREAD)
@Tag(TagNames.LARGE_RESOURCES)
public class LongTests extends BaseNd4jTestWithBackends {
DataType initialType = Nd4j.dataType();
@BeforeEach
public void beforeEach() {
System.gc();
}
@AfterEach
public void afterEach() {
System.gc();
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
public void testSomething1(Nd4jBackend backend) {
// we create 2D array, total nr. of elements is 2.4B elements, > MAX_INT
INDArray huge = Nd4j.create(8000000, 300);
INDArray huge = Nd4j.create(DataType.INT8,8000000, 300);
// we apply element-wise scalar ops, just to make sure stuff still works
huge.subi(0.5).divi(2);
huge.subi(1).divi(2);
// now we're checking different rows, they should NOT equal
@ -86,10 +97,10 @@ public class LongTests extends BaseNd4jTestWithBackends {
@Tag(TagNames.LONG_TEST)
public void testSomething2(Nd4jBackend backend) {
// we create 2D array, total nr. of elements is 2.4B elements, > MAX_INT
INDArray huge = Nd4j.create(100, 10);
INDArray huge = Nd4j.create(DataType.INT8,100, 10);
// we apply element-wise scalar ops, just to make sure stuff still works
huge.subi(0.5).divi(2);
huge.subi(1).divi(2);
// now we're checking different rows, they should NOT equal
@ -113,7 +124,7 @@ public class LongTests extends BaseNd4jTestWithBackends {
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
public void testLongTadOffsets1(Nd4jBackend backend) {
INDArray huge = Nd4j.create(230000000, 10);
INDArray huge = Nd4j.create(DataType.INT8,230000000, 10);
Pair<DataBuffer, DataBuffer> tad = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(huge, 1);
@ -125,10 +136,10 @@ public class LongTests extends BaseNd4jTestWithBackends {
@Tag(TagNames.LONG_TEST)
public void testLongTadOp1(Nd4jBackend backend) {
double exp = Transforms.manhattanDistance(Nd4j.create(1000).assign(1.0), Nd4j.create(1000).assign(2.0));
double exp = Transforms.manhattanDistance(Nd4j.create(DataType.INT16,1000).assign(1.0), Nd4j.create(DataType.INT16,1000).assign(2.0));
INDArray hugeX = Nd4j.create(2200000, 1000).assign(1.0);
INDArray hugeY = Nd4j.create(1, 1000).assign(2.0);
INDArray hugeX = Nd4j.create(DataType.INT16,2200000, 1000).assign(1.0);
INDArray hugeY = Nd4j.create(DataType.INT16,1, 1000).assign(2.0);
for (int x = 0; x < hugeX.rows(); x++) {
assertEquals(1000, hugeX.getRow(x).sumNumber().intValue(),"Failed at row " + x);
@ -144,9 +155,8 @@ public class LongTests extends BaseNd4jTestWithBackends {
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
public void testLongTadOp2(Nd4jBackend backend) {
INDArray hugeX = Nd4j.create(2300000, 1000).assign(1.0);
hugeX.addiRowVector(Nd4j.create(1000).assign(2.0));
INDArray hugeX = Nd4j.create(DataType.INT16,2300000, 1000).assign(1.0);
hugeX.addiRowVector(Nd4j.create(DataType.INT16,1000).assign(2.0));
for (int x = 0; x < hugeX.rows(); x++) {
assertEquals( hugeX.getRow(x).sumNumber().intValue(),3000,"Failed at row " + x);
@ -158,8 +168,8 @@ public class LongTests extends BaseNd4jTestWithBackends {
@Tag(TagNames.LONG_TEST)
public void testLongTadOp2_micro(Nd4jBackend backend) {
INDArray hugeX = Nd4j.create(230, 1000).assign(1.0);
hugeX.addiRowVector(Nd4j.create(1000).assign(2.0));
INDArray hugeX = Nd4j.create(DataType.INT16,230, 1000).assign(1.0);
hugeX.addiRowVector(Nd4j.create(DataType.INT16,1000).assign(2.0));
for (int x = 0; x < hugeX.rows(); x++) {
assertEquals( 3000, hugeX.getRow(x).sumNumber().intValue(),"Failed at row " + x);
@ -171,7 +181,7 @@ public class LongTests extends BaseNd4jTestWithBackends {
@Tag(TagNames.LONG_TEST)
public void testLongTadOp3(Nd4jBackend backend) {
INDArray hugeX = Nd4j.create(2300000, 1000).assign(1.0);
INDArray hugeX = Nd4j.create(DataType.INT16,2300000, 1000).assign(1.0);
INDArray mean = hugeX.mean(1);
for (int x = 0; x < hugeX.rows(); x++) {
@ -184,7 +194,7 @@ public class LongTests extends BaseNd4jTestWithBackends {
@Tag(TagNames.LONG_TEST)
public void testLongTadOp4(Nd4jBackend backend) {
INDArray hugeX = Nd4j.create(2300000, 1000).assign(1.0);
INDArray hugeX = Nd4j.create(DataType.INT8,2300000, 1000).assign(1.0);
INDArray mean = hugeX.argMax(1);
for (int x = 0; x < hugeX.rows(); x++) {
@ -199,7 +209,7 @@ public class LongTests extends BaseNd4jTestWithBackends {
List<INDArray> list = new ArrayList<>();
for (int i = 0; i < 2300000; i++) {
list.add(Nd4j.create(1000).assign(2.0));
list.add(Nd4j.create(DataType.INT8,1000).assign(2.0));
}
INDArray hugeX = Nd4j.vstack(list);

9
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/BasicWorkspaceTests.java
View File

@ -23,6 +23,8 @@ package org.nd4j.linalg.workspace;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -54,6 +56,7 @@ import static org.nd4j.linalg.api.buffer.DataType.DOUBLE;
@Slf4j
@Tag(TagNames.WORKSPACES)
@NativeTag
@Execution(ExecutionMode.SAME_THREAD)
public class BasicWorkspaceTests extends BaseNd4jTestWithBackends {
DataType initialType = Nd4j.dataType();
@ -959,6 +962,7 @@ public class BasicWorkspaceTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Execution(ExecutionMode.SAME_THREAD)
public void testMmap1(Nd4jBackend backend) {
// we don't support MMAP on cuda yet
if (Nd4j.getExecutioner().getClass().getName().toLowerCase().contains("cuda"))
@ -989,12 +993,13 @@ public class BasicWorkspaceTests extends BaseNd4jTestWithBackends {
}
@Test
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Execution(ExecutionMode.SAME_THREAD)
@Disabled("Still failing even with single thread execution")
public void testMmap2(Nd4jBackend backend) throws Exception {
// we don't support MMAP on cuda yet
if (Nd4j.getExecutioner().getClass().getName().toLowerCase().contains("cuda"))
if (!backend.getEnvironment().isCPU())
return;
File tmp = File.createTempFile("tmp", "fdsfdf");

6
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/CyclicWorkspaceTests.java
View File

@ -20,6 +20,7 @@
package org.nd4j.linalg.workspace;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.junit.jupiter.api.Disabled;
@ -65,8 +66,11 @@ public class CyclicWorkspaceTests extends BaseNd4jTestWithBackends {
}
}
@SneakyThrows
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public void testGc(Nd4jBackend backend) {
val indArray = Nd4j.create(4, 4);
indArray.putRow(0, Nd4j.create(new float[]{0, 2, -2, 0}));
@ -76,7 +80,7 @@ public class CyclicWorkspaceTests extends BaseNd4jTestWithBackends {
for (int i = 0; i < 100000000; i++) {
indArray.getRow(i % 3);
//Thread.sleep(1);
Thread.sleep(1);
}
}

4
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/DebugModeTests.java
View File

@ -26,6 +26,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -48,8 +50,8 @@ import static org.junit.jupiter.api.Assertions.*;
@Slf4j
@Tag(TagNames.WORKSPACES)
@NativeTag
@Execution(ExecutionMode.SAME_THREAD)
public class DebugModeTests extends BaseNd4jTestWithBackends {
DataType initialType = Nd4j.dataType();

50
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/SpecialWorkspaceTests.java
View File

@ -26,6 +26,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -62,12 +64,11 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
public void shutUp() {
Nd4j.getMemoryManager().setCurrentWorkspace(null);
Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread();
Nd4j.setDataType(this.initialType);
}
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled
@Execution(ExecutionMode.SAME_THREAD)
public void testVariableTimeSeries1(Nd4jBackend backend) {
WorkspaceConfiguration configuration = WorkspaceConfiguration
.builder()
@ -80,28 +81,28 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
.build();
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS1")) {
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
}
Nd4jWorkspace workspace = (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread("WS1");
assertEquals(0, workspace.getStepNumber());
long requiredMemory = 1000 * Nd4j.sizeOfDataType();
long requiredMemory = 1000 * DataType.DOUBLE.width();
long shiftedSize = ((long) (requiredMemory * 1.3)) + (8 - (((long) (requiredMemory * 1.3)) % 8));
assertEquals(requiredMemory, workspace.getSpilledSize());
assertEquals(shiftedSize, workspace.getInitialBlockSize());
assertEquals(workspace.getInitialBlockSize() * 4, workspace.getCurrentSize());
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace("WS1")) {
Nd4j.create(2000);
Nd4j.create(DataType.DOUBLE,2000);
}
assertEquals(0, workspace.getStepNumber());
assertEquals(1000 * Nd4j.sizeOfDataType(), workspace.getSpilledSize());
assertEquals(2000 * Nd4j.sizeOfDataType(), workspace.getPinnedSize());
assertEquals(1000 * DataType.DOUBLE.width(), workspace.getSpilledSize());
assertEquals(2000 * DataType.DOUBLE.width(), workspace.getPinnedSize());
assertEquals(0, workspace.getDeviceOffset());
@ -116,8 +117,8 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
for (int e = 0; e < 4; e++) {
for (int i = 0; i < 4; i++) {
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS1")) {
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
}
assertEquals((i + 1) * workspace.getInitialBlockSize(),
@ -144,9 +145,9 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
// we just do huge loop now, with pinned stuff in it
for (int i = 0; i < 100; i++) {
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS1")) {
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
assertEquals(1500 * Nd4j.sizeOfDataType(), workspace.getThisCycleAllocations());
}
@ -160,8 +161,8 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
// and we do another clean loo, without pinned stuff in it, to ensure all pinned allocates are gone
for (int i = 0; i < 100; i++) {
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS1")) {
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
}
}
@ -186,13 +187,12 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
// workspace.enableDebug(true);
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS1")) {
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
}
assertEquals(0, workspace.getStepNumber());
long requiredMemory = 1000 * Nd4j.sizeOfDataType();
long requiredMemory = 1000 * DataType.DOUBLE.width();
long shiftedSize = ((long) (requiredMemory * 1.3)) + (8 - (((long) (requiredMemory * 1.3)) % 8));
assertEquals(requiredMemory, workspace.getSpilledSize());
assertEquals(shiftedSize, workspace.getInitialBlockSize());
@ -200,9 +200,9 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
for (int i = 0; i < 100; i++) {
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS1")) {
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
Nd4j.create(DataType.DOUBLE,500);
}
}
@ -226,11 +226,11 @@ public class SpecialWorkspaceTests extends BaseNd4jTestWithBackends {
Nd4jWorkspace workspace =
(Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(configuration, "WS109");
INDArray row = Nd4j.linspace(1, 10, 10);
INDArray exp = Nd4j.create(10).assign(2.0);
INDArray row = Nd4j.linspace(1, 10, 10).castTo(DataType.DOUBLE);
INDArray exp = Nd4j.create(DataType.DOUBLE,10).assign(2.0);
INDArray result = null;
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "WS109")) {
INDArray matrix = Nd4j.create(10, 10);
INDArray matrix = Nd4j.create(DataType.DOUBLE,10, 10);
for (int e = 0; e < matrix.rows(); e++)
matrix.getRow(e).assign(row);

40
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/WorkspaceProviderTests.java
View File

@ -26,6 +26,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -57,6 +59,7 @@ import static org.junit.jupiter.api.Assertions.*;
@Slf4j
@Tag(TagNames.WORKSPACES)
@NativeTag
@Execution(ExecutionMode.SAME_THREAD)
public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
private static final WorkspaceConfiguration basicConfiguration = WorkspaceConfiguration.builder().initialSize(81920)
.overallocationLimit(0.1).policySpill(SpillPolicy.EXTERNAL).policyLearning(LearningPolicy.NONE)
@ -119,7 +122,6 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
public void shutUp() {
Nd4j.getMemoryManager().setCurrentWorkspace(null);
Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread();
Nd4j.setDataType(this.initialType);
}
/**
@ -144,7 +146,7 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
for (int x = 0; x < 100; x++) {
try (Nd4jWorkspace wsI = (Nd4jWorkspace) Nd4j.getWorkspaceManager()
.getWorkspaceForCurrentThread(configuration, "ITER").notifyScopeEntered()) {
INDArray array = Nd4j.create(100);
INDArray array = Nd4j.create(DataType.DOUBLE,100);
}
// only checking after workspace is initialized
@ -174,7 +176,7 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
try (Nd4jWorkspace ws1 = (Nd4jWorkspace) Nd4j.getWorkspaceManager()
.getWorkspaceForCurrentThread(configuration, "ITER").notifyScopeEntered()) {
INDArray array = Nd4j.create(100);
INDArray array = Nd4j.create(DataType.DOUBLE,100);
}
Nd4jWorkspace ws1 = (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(configuration,
@ -200,7 +202,7 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testMultithreading1() throws Exception {
public void testMultithreading1(Nd4jBackend backend) throws Exception {
final List<MemoryWorkspace> workspaces = new CopyOnWriteArrayList<>();
Nd4j.getWorkspaceManager().setDefaultWorkspaceConfiguration(basicConfiguration);
@ -283,21 +285,23 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
@Disabled
@Tag(TagNames.NEEDS_VERIFY)
public void testNestedWorkspacesOverlap1(Nd4jBackend backend) {
Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
Nd4j.getWorkspaceManager().setDefaultWorkspaceConfiguration(basicConfiguration);
try (Nd4jWorkspace ws1 = (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread("WS1").notifyScopeEntered()) {
INDArray array = Nd4j.create(new float[] {1f, 2f, 3f, 4f, 5f});
long reqMem = 5 * Nd4j.sizeOfDataType();
assertEquals(reqMem + (Nd4jWorkspace.alignmentBase - reqMem % Nd4jWorkspace.alignmentBase), ws1.getPrimaryOffset());
long reqMem = 5 * array.dataType().width();
long add = ((Nd4jWorkspace.alignmentBase / 2) - reqMem % (Nd4jWorkspace.alignmentBase / 2));
assertEquals(reqMem + add, ws1.getPrimaryOffset());
try (Nd4jWorkspace ws2 = (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread("WS2").notifyScopeEntered()) {
INDArray array2 = Nd4j.create(new float[] {1f, 2f, 3f, 4f, 5f});
reqMem = 5 * Nd4j.sizeOfDataType();
assertEquals(reqMem + (Nd4jWorkspace.alignmentBase - reqMem % Nd4jWorkspace.alignmentBase), ws1.getPrimaryOffset());
assertEquals(reqMem + (Nd4jWorkspace.alignmentBase - reqMem % Nd4jWorkspace.alignmentBase), ws2.getPrimaryOffset());
reqMem = 5 * array2.dataType().width();
assertEquals(reqMem + ((Nd4jWorkspace.alignmentBase / 2) - reqMem % (Nd4jWorkspace.alignmentBase / 2)), ws1.getPrimaryOffset());
assertEquals(reqMem + ((Nd4jWorkspace.alignmentBase / 2) - reqMem % (Nd4jWorkspace.alignmentBase / 2)), ws2.getPrimaryOffset());
try (Nd4jWorkspace ws3 = (Nd4jWorkspace) Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread("WS1")
.notifyScopeBorrowed()) {
@ -305,8 +309,8 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
INDArray array3 = Nd4j.create(new float[] {1f, 2f, 3f, 4f, 5f});
assertEquals(reqMem + (Nd4jWorkspace.alignmentBase - reqMem % Nd4jWorkspace.alignmentBase), ws2.getPrimaryOffset());
assertEquals((reqMem + (Nd4jWorkspace.alignmentBase - reqMem % Nd4jWorkspace.alignmentBase)) * 2, ws1.getPrimaryOffset());
assertEquals(reqMem + ((Nd4jWorkspace.alignmentBase / 2) - reqMem % (Nd4jWorkspace.alignmentBase / 2)), ws2.getPrimaryOffset());
assertEquals((reqMem + ((Nd4jWorkspace.alignmentBase / 2) - reqMem % (Nd4jWorkspace.alignmentBase / 2))) * 2, ws1.getPrimaryOffset());
}
}
}
@ -317,7 +321,7 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testWorkspacesSerde3() throws Exception {
INDArray array = Nd4j.create(10).assign(1.0);
INDArray array = Nd4j.create(DataType.DOUBLE,10).assign(1.0);
INDArray restored = null;
ByteArrayOutputStream bos = new ByteArrayOutputStream();
@ -600,7 +604,7 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testReallocate1(Nd4jBackend backend) {
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(reallocateConfiguration, "WS_1")) {
INDArray array = Nd4j.create(100);
INDArray array = Nd4j.create(DataType.DOUBLE,100);
}
@ -612,7 +616,7 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
assertEquals(100 * Nd4j.sizeOfDataType(), workspace.getCurrentSize());
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(reallocateConfiguration, "WS_1")) {
INDArray array = Nd4j.create(1000);
INDArray array = Nd4j.create(DataType.DOUBLE,1000);
}
assertEquals(1000 * Nd4j.sizeOfDataType(), workspace.getMaxCycleAllocations());
@ -634,14 +638,14 @@ public class WorkspaceProviderTests extends BaseNd4jTestWithBackends {
public void testNestedWorkspaces11(Nd4jBackend backend) {
for (int x = 1; x < 10; x++) {
try (MemoryWorkspace ws1 = Nd4j.getWorkspaceManager().getAndActivateWorkspace(basicConfiguration, "WS_1")) {
INDArray array1 = Nd4j.create(100 * x);
INDArray array1 = Nd4j.create(DataType.DOUBLE,100 * x);
for (int i = 1; i < 10; i++) {
try (MemoryWorkspace ws2 = Nd4j.getWorkspaceManager().getAndActivateWorkspace(basicConfiguration, "WS_1")) {
INDArray array2 = Nd4j.create(100 * x);
INDArray array2 = Nd4j.create(DataType.DOUBLE,100 * x);
for (int e = 1; e < 10; e++) {
try (MemoryWorkspace ws3 = Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(basicConfiguration, "WS_1").notifyScopeBorrowed()) {
INDArray array3 = Nd4j.create(100 * x);
INDArray array3 = Nd4j.create(DataType.DOUBLE,100 * x);
}
}
}

4
nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java
View File

@ -55,7 +55,7 @@ public abstract class BaseND4JTest {
* Override this method to set the default timeout for methods in the test class
*/
public long getTimeoutMilliseconds(){
return 90_000;
return 180_000;
}
/**
@ -95,7 +95,7 @@ public abstract class BaseND4JTest {
/**
* @return True if integration tests maven profile is enabled, false otherwise.
*/
public boolean isIntegrationTests(){
public boolean isIntegrationTests() {
if(integrationTest == null){
String prop = System.getenv("DL4J_INTEGRATION_TESTS");
integrationTest = Boolean.parseBoolean(prop);

38
nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/tags/ExpensiveTest.java Normal file
View File

@ -0,0 +1,38 @@
/*
*
* * ******************************************************************************
* * *
* * *
* * * This program and the accompanying materials are made available under the
* * * terms of the Apache License, Version 2.0 which is available at
* * * https://www.apache.org/licenses/LICENSE-2.0.
* * *
* * * See the NOTICE file distributed with this work for additional
* * * information regarding copyright ownership.
* * * Unless required by applicable law or agreed to in writing, software
* * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * * License for the specific language governing permissions and limitations
* * * under the License.
* * *
* * * SPDX-License-Identifier: Apache-2.0
* * *****************************************************************************
*
*
*/
package org.nd4j.common.tests.tags;
import org.junit.jupiter.api.Tag;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Tag(TagNames.LARGE_RESOURCES)
@Tag(TagNames.LONG_TEST)
public @interface ExpensiveTest {
}

1
nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/tags/TagNames.java
View File

@ -50,4 +50,5 @@ public class TagNames {
public final static String PYTHON = "python";
public final static String LONG_TEST = "long-running-test";
public final static String NEEDS_VERIFY = "needs-verify"; //tests that need verification of issue
public final static String LARGE_RESOURCES = "large-resources";
}

8
nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/pom.xml
View File

@ -106,6 +106,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<environmentVariables>
@ -116,7 +117,8 @@
<include>*.java</include>
<include>**/*.java</include>
</includes>
<argLine> -Xmx8g </argLine>
<argLine>-Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
</configuration>
</plugin>
</plugins>
@ -140,9 +142,11 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<argLine>-Xmx8g</argLine>
<argLine>-Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
</configuration>
</plugin>
</plugins>

4
nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/pom.xml
View File

@ -98,6 +98,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
<forkCount>${cpu.core.count}</forkCount>
<reuseForks>false</reuseForks>
<testSourceDirectory>src/test/java</testSourceDirectory>
@ -105,7 +106,8 @@
<include>*.java</include>
<include>**/*.java</include>
</includes>
<argLine> </argLine>
<argLine>-Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size}</argLine>
</configuration>
</plugin>
</plugins>

36
nd4j/nd4j-serde/nd4j-kryo/src/test/java/org/nd4j/TestNd4jKryoSerialization.java
View File

@ -20,35 +20,61 @@
package org.nd4j;
import com.sun.jna.Platform;
import lombok.AllArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.serializer.SerializerInstance;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.*;
import org.nd4j.common.primitives.*;
import org.nd4j.common.resources.Downloader;
import org.nd4j.common.tests.BaseND4JTest;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import scala.Tuple2;
import java.io.File;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Disabled("Ignoring due to flaky nature of tests")
@Slf4j
@Tag(TagNames.SPARK)
@Tag(TagNames.DIST_SYSTEMS)
public class TestNd4jKryoSerialization extends BaseND4JTest {
private JavaSparkContext sc;
@BeforeAll
@SneakyThrows
public static void beforeAll() {
if(Platform.isWindows()) {
File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp");
File binDir = new File(hadoopHome,"bin");
if(!binDir.exists())
binDir.mkdirs();
File outputFile = new File(binDir,"winutils.exe");
if(!outputFile.exists()) {
log.info("Fixing spark for windows");
Downloader.download("winutils.exe",
URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(),
outputFile,"db24b404d2331a1bec7443336a5171f1",3);
}
System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath());
}
}
@BeforeEach
public void before() {
SparkConf sparkConf = new SparkConf();

6
nd4j/samediff-import/samediff-import-onnx/pom.xml
View File

@ -49,6 +49,12 @@
<dependencies>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-common-tests</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>samediff-import-api</artifactId>

3
nd4j/samediff-import/samediff-import-onnx/src/test/kotlin/org/nd4j/samediff/frameworkimport/onnx/modelzoo/TestPretrainedModels.kt
View File

@ -40,6 +40,7 @@ import org.apache.commons.io.FileUtils
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import org.nd4j.common.resources.Downloader
import org.nd4j.common.tests.tags.ExpensiveTest
import org.nd4j.common.util.ArchiveUtils
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.samediff.frameworkimport.onnx.importer.OnnxFrameworkImporter
@ -50,7 +51,7 @@ import java.io.File
import java.net.URI
data class InputDataset(val dataSetIndex: Int,val inputPaths: List<String>,val outputPaths: List<String>)
@Disabled
@ExpensiveTest
class TestPretrainedModels {
val modelBaseUrl = "https://media.githubusercontent.com/media/onnx/models/master"

5
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java
View File

@ -22,6 +22,7 @@ package org.nd4j.imports.tfgraphs;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -33,6 +34,7 @@ import org.nd4j.autodiff.samediff.transform.OpPredicate;
import org.nd4j.autodiff.samediff.transform.SubGraph;
import org.nd4j.autodiff.samediff.transform.SubGraphPredicate;
import org.nd4j.autodiff.samediff.transform.SubGraphProcessor;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.graph.ui.LogFileWriter;
import org.nd4j.imports.graphmapper.tf.TFGraphMapper;
import org.nd4j.imports.tensorflow.TFImportOverride;
@ -55,7 +57,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Slf4j
@Disabled("AB 2019/05/21 - JVM Crash on linux-x86_64-cuda-9.2, linux-ppc64le-cpu - Issue #7657")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class BERTGraphTest extends BaseNd4jTestWithBackends {

2
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/CustomOpTests.java
View File

@ -44,7 +44,7 @@ public class CustomOpTests extends BaseNd4jTestWithBackends {
@ParameterizedTest
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
public void testPad(Nd4jBackend backend){
public void testPad(Nd4jBackend backend) {
INDArray in = Nd4j.create(DataType.FLOAT, 1, 28, 28, 264);
INDArray pad = Nd4j.createFromArray(new int[][]{{0,0},{0,1},{0,1},{0,0}});

4
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllLibnd4j.java
View File

@ -27,6 +27,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.params.provider.Arguments;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
@ -41,7 +42,8 @@ import java.util.stream.Stream;
@Slf4j
@Disabled("AB 2019/05/21 - JVM Crashes - Issue #7657")
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class TFGraphTestAllLibnd4j { //Note: Can't extend BaseNd4jTest here as we need no-arg constructor for parameterized tests
private Map<String, INDArray> inputs;

3
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestAllSameDiff.java
View File

@ -26,6 +26,7 @@ import org.junit.jupiter.api.*;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
@ -38,6 +39,8 @@ import java.util.*;
import java.util.stream.Stream;
@Slf4j
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here as we need no-arg constructor for parameterized tests

4
nd4j/samediff-import/samediff-import-tensorflow/src/test/java/org/nd4j/imports/tfgraphs/TFGraphTestList.java
View File

@ -29,6 +29,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.nd4j.common.tests.tags.TagNames;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
import org.nd4j.linalg.factory.Nd4j;
@ -44,7 +45,8 @@ import java.util.Map;
import java.util.stream.Stream;
@Disabled
@Tag(TagNames.LONG_TEST)
@Tag(TagNames.LARGE_RESOURCES)
public class TFGraphTestList {

Some files were not shown because too many files have changed in this diff Show More