From 5e55e92002d9611a6fcdf6d1e9e4de57960b0d83 Mon Sep 17 00:00:00 2001 From: Alex Black Date: Wed, 13 May 2020 01:37:11 +1000 Subject: [PATCH 1/8] Empty array casting fix (#457) * Empty array casting fix Signed-off-by: Alex Black * Tests Signed-off-by: Alex Black --- .../nd4j/linalg/api/ndarray/BaseNDArray.java | 2 +- .../test/java/org/nd4j/linalg/Nd4jTestsC.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java index 835a2f4cb..052251734 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java @@ -5521,7 +5521,7 @@ public abstract class BaseNDArray implements INDArray, Iterable { public INDArray castTo(DataType dataType) { if(dataType == dataType()) //No-op if correct datatype return this; - if(isEmpty()){ + if(isEmpty() && rank() == 0){ return Nd4j.empty(dataType); } val result = Nd4j.createUninitialized(dataType, this.shape(), this.ordering()); diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java index da8983118..46f47017e 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java @@ -8414,6 +8414,26 @@ public class Nd4jTestsC extends BaseNd4jTest { } } + + @Test + public void testShape0Casts(){ + for(DataType dt : DataType.values()){ + if(!dt.isNumerical()) + continue; + + INDArray a1 = Nd4j.create(dt, 1,0,2); + + for(DataType dt2 : DataType.values()){ + if(!dt2.isNumerical()) + continue; + INDArray a2 = a1.castTo(dt2); + + assertArrayEquals(a1.shape(), a2.shape()); + assertEquals(dt2, a2.dataType()); + } + } + } + @Override public char ordering() { return 'c'; From f547f783d95c2aa41bd5bac1ec25611926d3b3db Mon Sep 17 00:00:00 2001 From: Alex Black Date: Wed, 13 May 2020 14:50:23 +1000 Subject: [PATCH 2/8] Keras import - remove debug lines (println/log) [WIP] (#459) * Remove debug line from KerasConvolution2D Signed-off-by: Alex Black * Remove more debug lines Signed-off-by: Alex Black --- .../deeplearning4j/nn/modelimport/keras/layers/KerasInput.java | 2 -- .../nn/modelimport/keras/layers/TFOpLayerImpl.java | 2 +- .../keras/layers/convolutional/KerasConvolution2D.java | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java index d83d27011..530d79d1c 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java +++ b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java @@ -128,8 +128,6 @@ public class KerasInput extends KerasLayer { break; case 2: if(this.dimOrder != null) { - System.out.println("Dim order: " + this.dimOrder); - System.out.println("Input shape: " + ArrayUtils.toString(this.inputShape)); switch (this.dimOrder) { case TENSORFLOW: //NWC == channels_last myInputType = new InputType.InputTypeRecurrent(this.inputShape[1], this.inputShape[0], RNNFormat.NWC); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java index 751f939c9..47d063826 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java +++ b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java @@ -103,7 +103,7 @@ public class TFOpLayerImpl extends AbstractLayer { String dtype = inputDataTypes.get(inpName); graph = "node{\nname: \"" + inpName + "\"\nop: \"Placeholder\"\nattr{\nkey: \"dtype\"\n value {\n type: " + dtype + "}\n}\n}\n" + graph; } - log.info(graph); + //log.info(graph); GraphDef.Builder graphDefBuilder = GraphDef.newBuilder(); TextFormat.getParser().merge(graph, graphDefBuilder); GraphDef graphDef = graphDefBuilder.build(); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java index 51d481e5d..84c24614a 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java +++ b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java @@ -95,7 +95,6 @@ public class KerasConvolution2D extends KerasConvolution { LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion); - System.out.println("----" + dimOrder); ConvolutionLayer.Builder builder = new ConvolutionLayer.Builder().name(this.layerName) .nOut(getNOutFromConfig(layerConfig, conf)).dropOut(this.dropout) .activation(getIActivationFromConfig(layerConfig, conf)) From c396fcb9605999e6586b95d460bd2524e0f8589e Mon Sep 17 00:00:00 2001 From: raver119 Date: Wed, 13 May 2020 08:12:07 +0300 Subject: [PATCH 3/8] More pre-release fixes (#456) * - numPrefixBlocks fix for threshold_encoding - temparrays pointers fixed Signed-off-by: raver119@gmail.com * auto configuration of memory workspace for gradients sharing Signed-off-by: raver119@gmail.com * limit sparse encoding message size Signed-off-by: raver119@gmail.com * one more workspace test Signed-off-by: raver119@gmail.com * one more CUDA-specific test Signed-off-by: raver119@gmail.com * one more CUDA-specific workspace test Signed-off-by: raver119@gmail.com * one more CUDA-specific workspace test Signed-off-by: raver119@gmail.com * one more CUDA-specific workspace test Signed-off-by: raver119@gmail.com * add separate host/device reset for circular workspace mode Signed-off-by: raver119@gmail.com * new PW builder method for encoder memory amount Signed-off-by: raver119@gmail.com * "inplace" execution for threshold encoding Signed-off-by: raver119@gmail.com --- .../parallelism/ParallelWrapper.java | 32 +++++- .../helpers/cuda/compression/threshold.cu | 9 +- .../layers_tests/DeclarableOpsTests19.cpp | 47 ++++++++ .../api/ops/compression/EncodeThreshold.java | 6 + .../ops/executioner/DefaultOpExecutioner.java | 26 ++++- .../nd4j/jita/workspace/CudaWorkspace.java | 9 +- .../jita/workspace/CudaWorkspaceTest.java | 105 ++++++++++++++++++ .../java/org/nd4j/nativeblas/Nd4jCpu.java | 15 +++ .../linalg/workspace/BasicWorkspaceTests.java | 25 +++++ 9 files changed, 264 insertions(+), 10 deletions(-) create mode 100644 nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/test/java/org/nd4j/jita/workspace/CudaWorkspaceTest.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java index 00ca839f7..842ac34b2 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java +++ b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java @@ -20,6 +20,8 @@ import lombok.*; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.core.storage.StatsStorageRouter; import org.deeplearning4j.core.storage.listener.RoutingIterationListener; +import org.deeplearning4j.optimize.solvers.accumulation.EncodingHandler; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.AdaptiveThresholdAlgorithm; import org.nd4j.linalg.dataset.AsyncDataSetIterator;; import org.nd4j.linalg.dataset.AsyncMultiDataSetIterator; import org.deeplearning4j.datasets.iterator.DummyBlockDataSetIterator; @@ -688,6 +690,7 @@ public class ParallelWrapper implements AutoCloseable { protected Supplier updaterParamsSupplier; protected ThresholdAlgorithm thresholdAlgorithm; protected ResidualPostProcessor residualPostProcessor; + protected Long encoderMemory = -1L; protected GradientsAccumulator accumulator; @@ -872,6 +875,19 @@ public class ParallelWrapper implements AutoCloseable { return this; } + /** + * This method allows to define amount of temporary memory that will be used for gradients sharing. + * Typically it's safe to keep default value. + * + * Default value: -1, amount of temporary memory will be calculated automatically + * @param numBytes number of bytes to be used + * @return + */ + public Builder temporaryMemory(@NonNull Long numBytes) { + this.encoderMemory = numBytes; + return this; + } + /** * Set the residual post processor algorithm. Not used for single machine training (only for PW used in a * distributed setting), and should not be set by users in most cases. @@ -907,11 +923,23 @@ public class ParallelWrapper implements AutoCloseable { } break; case SHARED_GRADIENTS: { - Preconditions.checkState(thresholdAlgorithm != null, "Cannot use SHARED_GRADIENTS training mode without setting a threshold algorithm"); + if (thresholdAlgorithm == null) + thresholdAlgorithm = new AdaptiveThresholdAlgorithm(); + this.trainerContext = new SymmetricTrainerContext(); if (this.accumulator == null) { log.info("Creating new GradientsAccumulator instance with default threshold of [5e-4]"); - this.accumulator = new EncodedGradientsAccumulator(workers, thresholdAlgorithm, residualPostProcessor, false); + val numParams = model.numParams(); + + // we're limiting max size of updates for Sparse encoding to the size of bitmap encoded message + val maxUpdate = (int) (numParams / 16 + 5); + + // memory sie in number of bytes + long memorySize = encoderMemory == null || encoderMemory < 0 + ? maxUpdate * 4 * (workers + 3) + : encoderMemory; + + this.accumulator = new EncodedGradientsAccumulator(workers, new EncodingHandler(thresholdAlgorithm, residualPostProcessor, maxUpdate, false), memorySize, workers + 2, Integer.MAX_VALUE, false); } } break; diff --git a/libnd4j/include/ops/declarable/helpers/cuda/compression/threshold.cu b/libnd4j/include/ops/declarable/helpers/cuda/compression/threshold.cu index 19cd3f67c..bbe6d6881 100644 --- a/libnd4j/include/ops/declarable/helpers/cuda/compression/threshold.cu +++ b/libnd4j/include/ops/declarable/helpers/cuda/compression/threshold.cu @@ -163,14 +163,13 @@ namespace sd { // here we just calculate number of sumBlock arrays do { int numPrefixBlocks = sd::math::nd4j_max(1, sd::math::nd4j_ceil((float) numElts / (2.0f * prefixThreads))); - if (numBlocks > 1) { + if (numPrefixBlocks > 1) { level++; } numElts = numPrefixBlocks; } while (numElts > 1); - std::vector tempArrays(level); std::vector pointers(level); @@ -181,13 +180,13 @@ namespace sd { int numPrefixBlocks = sd::math::nd4j_max(1, sd::math::nd4j_ceil((float) numElts / (2.0f * prefixThreads))); if (numPrefixBlocks > 1) { tempArrays[level] = std::move(NDArrayFactory::create('c', {numPrefixBlocks})); - pointers[level] = tempArrays[level++].specialBuffer(); + pointers[level] = tempArrays[level].specialBuffer();; + level++; } numElts = numPrefixBlocks; } while (numElts > 1); PointersManager pm(LaunchContext::defaultContext(), "thresholdEncode"); - auto dptr = pm.replicatePointer(pointers.data(), pointers.size() * 8); auto offsets = NDArrayFactory::create('c', {numBlocks}); // we want to check, if we're hiting external limit on number of encoded elements @@ -200,7 +199,7 @@ namespace sd { NDArray::prepareSpecialUse({}, {&encoded, &updates}); // filling offsets - encodeThresholdP2Int_(reinterpret_cast(dptr), + encodeThresholdP2Int_(reinterpret_cast(pointers.data()), reinterpret_cast(blocks.specialBuffer()), numBlocks, reinterpret_cast(offsets.specialBuffer())); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests19.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests19.cpp index 641728ad3..ce5038020 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests19.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests19.cpp @@ -228,6 +228,53 @@ TEST_F(DeclarableOpsTests19, test_threshold_encode_decode) { ASSERT_EQ(exp, initial); } +TEST_F(DeclarableOpsTests19, test_threshold_encode_decode_2) { + // [2,1,135079944,1,1,8192,1,99] + auto initial = NDArrayFactory::create('c', {1, 135079944}); + initial = 1.0f; + auto exp = initial.dup(); + auto neg = initial.like(); + neg = 0.5f; + + sd::ops::encode_threshold enc; + auto enc_result = enc.evaluate({&initial}, {0.5f}); + auto encoded = enc_result.at(1); + + ASSERT_EQ(135079944 + 4, encoded->lengthOf()); + ASSERT_NE(exp, initial); +/* + for (int e = 0; e < initial.lengthOf(); e++) { + auto f = initial.e(e); + if (f != 0.5f) { + nd4j_printf("initial[%i] = %f\n", e, f); + throw std::runtime_error(""); + } + } + */ + ASSERT_EQ(neg, initial); + + // checking equality of all encoded bits + //for (int e = 5; e < encoded->lengthOf() - 1; e++) { + //if (encoded->e(e) != encoded->e(e - 1) + 1) + //nd4j_printf("Non equal encoded values at E[%i]: %i;\n", e, encoded->e(e)); + //} + + sd::ops::decode_threshold dec; + auto status = dec.execute({&initial, encoded}, {&initial}); + ASSERT_EQ(Status::OK(), status); + + // checking equality of all dedoded bits + /* + for (int e = 0; e < initial.lengthOf(); e++) { + auto f = initial.e(e); + if (f != 1.0f) + nd4j_printf("initial[%i] = %f\n", e, f); + } + */ + + ASSERT_EQ(exp, initial); +} + TEST_F(DeclarableOpsTests19, test_matmul_ccc) { auto x = NDArrayFactory::create('c', {10, 10}); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java index 621b459e9..fec9582be 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java @@ -41,6 +41,12 @@ public class EncodeThreshold extends DynamicCustomOp { this(updates, threshold, Integer.MAX_VALUE); } + public EncodeThreshold(@NonNull INDArray updates, @NonNull INDArray encoded, float threshold, @NonNull Integer boundary) { + this(updates, threshold, boundary); + + addOutputArgument(updates, encoded); + } + public EncodeThreshold(@NonNull INDArray updates, float threshold, @NonNull Integer boundary) { addInputArgument(updates); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java index af3ffb2c4..f4be7cc92 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java @@ -692,9 +692,33 @@ public abstract class DefaultOpExecutioner implements OpExecutioner { return thresholdEncode(input, threshold, Integer.MAX_VALUE); } + private long _length(long[] shape) { + // scalar case + if (shape.length == 0) + return 1; + else if (shape.length == 1) + return shape[0]; + else { + long length = 1; + for (int e = 0; e < shape.length; e++) + length *= shape[e]; + + return length; + } + } + @Override public INDArray thresholdEncode(INDArray input, double threshold, Integer boundary) { - val result = Nd4j.exec(new EncodeThreshold(input, (float) threshold, boundary))[1]; + val op_shape = new EncodeThreshold(input, (float) threshold, boundary); + val shapes = Nd4j.getExecutioner().calculateOutputShape(op_shape); + + if (_length(shapes.get(1).getShape()) < 2) + return null; + + val result = Nd4j.create(DataType.INT32, shapes.get(1).getShape()); + + op_shape.addOutputArgument(input, result); + Nd4j.exec(op_shape); return result.getInt(0) > 0 ? result : null; } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java index e8d9b2d18..0c4cf9ffa 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java @@ -191,7 +191,7 @@ public class CudaWorkspace extends Nd4jWorkspace { // spill if (workspaceConfiguration.getPolicyReset() == ResetPolicy.ENDOFBUFFER_REACHED && currentSize.get() > 0 && !trimmer && Nd4j.getWorkspaceManager().getDebugMode() != DebugMode.SPILL_EVERYTHING) { //log.info("End of space reached. Current offset: {}; requiredMemory: {}", deviceOffset.get(), requiredMemory); - reset(); + deviceOffset.set(0); resetPlanned.set(true); return alloc(requiredMemory, kind, type, initialize); } @@ -204,7 +204,6 @@ public class CudaWorkspace extends Nd4jWorkspace { if (isDebug.get()) { log.info("Workspace [{}] device_{}: spilled DEVICE array of {} bytes, capacity of {} elements", id, Nd4j.getAffinityManager().getDeviceForCurrentThread(), requiredMemory, numElements); } - //Nd4j.getWorkspaceManager().printAllocationStatisticsForCurrentThread(); val shape = new AllocationShape(requiredMemory / Nd4j.sizeOfDataType(type), Nd4j.sizeOfDataType(type), type); @@ -258,6 +257,12 @@ public class CudaWorkspace extends Nd4jWorkspace { return ptr; } else { // log.info("Spilled HOST array of {} bytes, capacity of {} elements", requiredMemory, numElements); + if (workspaceConfiguration.getPolicyReset() == ResetPolicy.ENDOFBUFFER_REACHED && currentSize.get() > 0 && !trimmer && Nd4j.getWorkspaceManager().getDebugMode() != DebugMode.SPILL_EVERYTHING) { + //log.info("End of space reached. Current offset: {}; requiredMemory: {}", deviceOffset.get(), requiredMemory); + hostOffset.set(0); + //resetPlanned.set(true); + return alloc(requiredMemory, kind, type, initialize); + } val shape = new AllocationShape(requiredMemory / Nd4j.sizeOfDataType(type), Nd4j.sizeOfDataType(type), type); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/test/java/org/nd4j/jita/workspace/CudaWorkspaceTest.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/test/java/org/nd4j/jita/workspace/CudaWorkspaceTest.java new file mode 100644 index 000000000..7b572de9c --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/test/java/org/nd4j/jita/workspace/CudaWorkspaceTest.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.k. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.jita.workspace; + +import lombok.val; +import org.junit.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.concurrency.AffinityManager; +import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; +import org.nd4j.linalg.api.memory.enums.AllocationPolicy; +import org.nd4j.linalg.api.memory.enums.LearningPolicy; +import org.nd4j.linalg.api.memory.enums.ResetPolicy; +import org.nd4j.linalg.api.memory.enums.SpillPolicy; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.Assert.*; + +public class CudaWorkspaceTest { + + @Test + public void testCircularWorkspaceAsymmetry_1() { + // circular workspace mode + val configuration = WorkspaceConfiguration.builder().initialSize(10 * 1024 * 1024) + .policyReset(ResetPolicy.ENDOFBUFFER_REACHED).policyAllocation(AllocationPolicy.STRICT) + .policySpill(SpillPolicy.FAIL).policyLearning(LearningPolicy.NONE).build(); + + + try (val ws = (CudaWorkspace) Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "circular_ws")) { + val array = Nd4j.create(DataType.FLOAT, 10, 10); + + assertEquals(0, ws.getHostOffset()); + assertNotEquals(0, ws.getDeviceOffset()); + + // we expect that this array has no data/buffer on HOST side + assertEquals(AffinityManager.Location.DEVICE, Nd4j.getAffinityManager().getActiveLocation(array)); + + // since this array doesn't have HOST buffer - it will allocate one now + array.getDouble(3L); + + assertEquals(ws.getHostOffset(), ws.getDeviceOffset()); + } + + try (val ws = (CudaWorkspace) Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "circular_ws")) { + assertEquals(ws.getHostOffset(), ws.getDeviceOffset()); + } + + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + } + + @Test + public void testCircularWorkspaceAsymmetry_2() { + // circular workspace mode + val configuration = WorkspaceConfiguration.builder().initialSize(10 * 1024 * 1024) + .policyReset(ResetPolicy.ENDOFBUFFER_REACHED).policyAllocation(AllocationPolicy.STRICT) + .policySpill(SpillPolicy.FAIL).policyLearning(LearningPolicy.NONE).build(); + + val root = Nd4j.create(DataType.FLOAT, 1000000).assign(119); + + for (int e = 0; e < 100; e++) { + try (val ws = (CudaWorkspace) Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "circular_ws")) { + val array = Nd4j.create(DataType.FLOAT, root.shape()); + array.assign(root); + + array.data().getInt(3); + + assertEquals(ws.getHostOffset(), ws.getDeviceOffset()); + } + } + } + + @Test + public void testCircularWorkspaceAsymmetry_3() { + // circular workspace mode + val configuration = WorkspaceConfiguration.builder().initialSize(10 * 1024 * 1024) + .policyReset(ResetPolicy.ENDOFBUFFER_REACHED).policyAllocation(AllocationPolicy.STRICT) + .policySpill(SpillPolicy.FAIL).policyLearning(LearningPolicy.NONE).build(); + + val root = Nd4j.create(DataType.FLOAT, 1000000).assign(119); + + for (int e = 0; e < 100; e++) { + try (val ws = (CudaWorkspace) Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "circular_ws")) { + val array = Nd4j.create(DataType.FLOAT, root.shape()); + array.assign(root); + + val second = Nd4j.create(DataType.FLOAT, root.shape()); + + array.data().getInt(3); + } + } + } +} \ No newline at end of file diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java index b67949a54..b97274ba1 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java @@ -16619,6 +16619,21 @@ public static final int TAD_THRESHOLD = TAD_THRESHOLD(); private native void allocate(); public native ShapeList calculateOutputShape(ShapeList inputShape, @ByRef Context block); } + @Namespace("sd::ops") public static class clipbyavgnorm_bp extends DeclarableCustomOp { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public clipbyavgnorm_bp(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public clipbyavgnorm_bp(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public clipbyavgnorm_bp position(long position) { + return (clipbyavgnorm_bp)super.position(position); + } + + public clipbyavgnorm_bp() { super((Pointer)null); allocate(); } + private native void allocate(); + public native ShapeList calculateOutputShape(ShapeList inputShape, @ByRef Context block); + } // #endif // #if NOT_EXCLUDED(OP_cumsum) diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/BasicWorkspaceTests.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/BasicWorkspaceTests.java index 5b357d9a4..290b69723 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/BasicWorkspaceTests.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/workspace/BasicWorkspaceTests.java @@ -26,6 +26,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.nd4j.linalg.BaseNd4jTest; import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.concurrency.AffinityManager; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; import org.nd4j.linalg.api.memory.enums.*; @@ -1219,6 +1220,30 @@ public class BasicWorkspaceTests extends BaseNd4jTest { Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); } + @Test + public void testCircularWorkspaceAsymmetry_1() { + // nothing to test on CPU here + if (Nd4j.getEnvironment().isCPU()) + return; + + // circular workspace mode + val configuration = WorkspaceConfiguration.builder().initialSize(10 * 1024 * 1024) + .policyReset(ResetPolicy.ENDOFBUFFER_REACHED).policyAllocation(AllocationPolicy.STRICT) + .policySpill(SpillPolicy.FAIL).policyLearning(LearningPolicy.NONE).build(); + + + try (val ws = Nd4j.getWorkspaceManager().getAndActivateWorkspace(configuration, "circular_ws")) { + val array = Nd4j.create(DataType.FLOAT, 10, 10); + + // we expect that this array has no data/buffer on HOST side + assertEquals(AffinityManager.Location.DEVICE, Nd4j.getAffinityManager().getActiveLocation(array)); + + // since this array doesn't have HOST buffer - it will allocate one now + array.getDouble(3L); + } + + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + } @Override public char ordering() { From a4d74ec4d0168dc4ed6dd3c68cddac2f5e7dc7f2 Mon Sep 17 00:00:00 2001 From: Alex Black Date: Wed, 13 May 2020 19:47:51 +1000 Subject: [PATCH 4/8] Fix wrong indexer for some DataBuffer constructors for UINT32 datatype (#458) Signed-off-by: Alex Black --- .../jcublas/buffer/BaseCudaDataBuffer.java | 8 ++- .../nativecpu/buffer/BaseCpuDataBuffer.java | 9 +++- .../test/java/org/nd4j/linalg/Nd4jTestsC.java | 50 +++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java index 14e64df61..1da7e1c36 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java @@ -1630,7 +1630,7 @@ public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCuda setIndexer(ShortIndexer.create((ShortPointer) pointer)); } else if (t == DataType.UINT32) { pointer = new PagedPointer(cptr, length).asIntPointer(); - setIndexer(IntIndexer.create((IntPointer) pointer)); + setIndexer(UIntIndexer.create((IntPointer) pointer)); } else if (t == DataType.INT) { pointer = new PagedPointer(cptr, length).asIntPointer(); setIndexer(IntIndexer.create((IntPointer) pointer)); @@ -1699,6 +1699,9 @@ public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCuda indexer = ShortIndexer.create((ShortPointer) pointer); break; case UINT32: + pointer = nPtr.asIntPointer(); + indexer = UIntIndexer.create((IntPointer) pointer); + break; case INT: pointer = nPtr.asIntPointer(); indexer = IntIndexer.create((IntPointer) pointer); @@ -1750,6 +1753,9 @@ public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCuda indexer = ShortIndexer.create((ShortPointer) pointer); break; case UINT32: + pointer = nPtr.asIntPointer(); + indexer = UIntIndexer.create((IntPointer) pointer); + break; case INT: pointer = nPtr.asIntPointer(); indexer = IntIndexer.create((IntPointer) pointer); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java index 7a2a8467a..6666068da 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java @@ -411,7 +411,7 @@ public abstract class BaseCpuDataBuffer extends BaseDataBuffer implements Deallo setIndexer(ShortIndexer.create((ShortPointer) pointer)); } else if (t == DataType.UINT32) { pointer = new PagedPointer(cptr, length).asIntPointer(); - setIndexer(IntIndexer.create((IntPointer) pointer)); + setIndexer(UIntIndexer.create((IntPointer) pointer)); } else if (t == DataType.INT) { pointer = new PagedPointer(cptr, length).asIntPointer(); setIndexer(IntIndexer.create((IntPointer) pointer)); @@ -514,7 +514,6 @@ public abstract class BaseCpuDataBuffer extends BaseDataBuffer implements Deallo attached = true; parentWorkspace = workspace; - // FIXME: need unsigned indexer here pointer = workspace.alloc(length * getElementSize(), dataType(), initialize).asIntPointer(); //new IntPointer(length()); setIndexer(UIntIndexer.create((IntPointer) pointer)); @@ -882,6 +881,9 @@ public abstract class BaseCpuDataBuffer extends BaseDataBuffer implements Deallo indexer = ShortIndexer.create((ShortPointer) pointer); break; case UINT32: + pointer = nPtr.asIntPointer(); + indexer = UIntIndexer.create((IntPointer) pointer); + break; case INT: pointer = nPtr.asIntPointer(); indexer = IntIndexer.create((IntPointer) pointer); @@ -932,6 +934,9 @@ public abstract class BaseCpuDataBuffer extends BaseDataBuffer implements Deallo indexer = ShortIndexer.create((ShortPointer) pointer); break; case UINT32: + pointer = nPtr.asIntPointer(); + indexer = UIntIndexer.create((IntPointer) pointer); + break; case INT: pointer = nPtr.asIntPointer(); indexer = IntIndexer.create((IntPointer) pointer); diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java index 46f47017e..a70ede362 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java @@ -8411,6 +8411,56 @@ public class Nd4jTestsC extends BaseNd4jTest { INDArray arr = Nd4j.create(db, new long[]{lengthElements}); arr.toStringFull(); + arr.toString(); + + for(DataType dt2 : DataType.values()) { + if (dt2 == DataType.COMPRESSED || dt2 == DataType.UTF8 || dt2 == DataType.UNKNOWN) + continue; + INDArray a2 = arr.castTo(dt2); + a2.toStringFull(); + } + } + } + + @Test + public void testCreateBufferFromByteBufferViews(){ + + for(DataType dt : DataType.values()){ + if(dt == DataType.COMPRESSED || dt == DataType.UTF8 || dt == DataType.UNKNOWN) + continue; +// System.out.println(dt); + + int lengthBytes = 256; + int lengthElements = lengthBytes / dt.width(); + ByteBuffer bb = ByteBuffer.allocateDirect(lengthBytes); + + DataBuffer db = Nd4j.createBuffer(bb, dt, lengthElements, 0); + INDArray arr = Nd4j.create(db, new long[]{lengthElements/2, 2}); + + arr.toStringFull(); + + INDArray view = arr.get(NDArrayIndex.all(), NDArrayIndex.point(0)); + INDArray view2 = arr.get(NDArrayIndex.point(1), NDArrayIndex.all()); + + view.toStringFull(); + view2.toStringFull(); + } + } + + @Test + public void testTypeCastingToString(){ + + for(DataType dt : DataType.values()) { + if (dt == DataType.COMPRESSED || dt == DataType.UTF8 || dt == DataType.UNKNOWN) + continue; + INDArray a1 = Nd4j.create(dt, 10); + for(DataType dt2 : DataType.values()) { + if (dt2 == DataType.COMPRESSED || dt2 == DataType.UTF8 || dt2 == DataType.UNKNOWN) + continue; + + INDArray a2 = a1.castTo(dt2); + a2.toStringFull(); + } } } From 3714214b8c2221354b0856294312b7027d6e4267 Mon Sep 17 00:00:00 2001 From: Andrii T <39699084+atuzhykov@users.noreply.github.com> Date: Wed, 13 May 2020 14:02:57 +0300 Subject: [PATCH 5/8] temporary ignored section added (#462) Signed-off-by: Andrii Tuzhykov --- .../TFGraphs/TFGraphTestAllSameDiff.java | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java index b76b45dcd..a20e6ea35 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java @@ -124,7 +124,60 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a "bitcast/from_float64_to_int64", "bitcast/from_rank2_float64_to_int64", "bitcast/from_float64_to_uint64" - }; + + + //NEWLY ADDED TESTCASES from 27/04/2020 + "non_max_suppression_v2/.*", "non_max_suppression/.*", + "random_gamma/.*", + "non_max_suppression_v5/.*", + "non_max_suppression_v4/.*", + "non_max_suppression_v3/.*", + "dropout/.*", + "max_pool_with_argmax/.*", + "conv2d_transpose/.*", + "Conv3DBackpropInputV2/.*", + "Conv3DBackpropInput/.*", + "mod/.*", + "leaky_relu/.*", + "DeepCopy/.*", + "empty/.*", + "ones_like/.*", + "is_non_decreasing/.*", + "div/.*", + "lgamma/.*", + "random_uniform/.*", + "random_uniform_int/.*", + "resize_area/.*", + "zeros_like_tf1/.*", + "Conv2DTranspose/.*", + "rgb_to_yuv/.*", + "rgb_to_grayscale/.*", + "rgb_to_yiq/.*", + "losses/.*", + "yiq_to_rgb/.*", + "yuv_to_rgb/.*", + "emptyArrayTests/.*", + "random_normal/.*", + "random_shuffle/.*", + "random_poisson_v2/.*", + "random_poisson/.*", + "random_crop/.*", + "compare_and_bitpack/.*", + "adjust_contrast/.*", + "confusion/.*", + "bitcast/.*", + "roll/.*", + "matrix_band_part/.*", + "conv3d_transpose_layers/.*", + "multinomial/.*", + "unsorted_segment/.*", + "cnn2d_nn/.*", + "truncatemod/.*", + "bincount/.*", + "slogdet/.*", + "adjust_contrast_v2/.*" + +}; /* As per TFGraphTestList.printArraysDebugging - this field defines a set of regexes for test cases that should have all arrays printed during execution. From 1ce65fced416dd4e44e4cedaab90575068c55e51 Mon Sep 17 00:00:00 2001 From: Alex Black Date: Wed, 13 May 2020 21:05:39 +1000 Subject: [PATCH 6/8] Fix missing comma (#463) Signed-off-by: Alex Black --- .../java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java index a20e6ea35..72c705852 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/TFGraphTestAllSameDiff.java @@ -123,7 +123,7 @@ public class TFGraphTestAllSameDiff { //Note: Can't extend BaseNd4jTest here a //AB 2020/01/07 - Known issues "bitcast/from_float64_to_int64", "bitcast/from_rank2_float64_to_int64", - "bitcast/from_float64_to_uint64" + "bitcast/from_float64_to_uint64", //NEWLY ADDED TESTCASES from 27/04/2020 From 60f103fb03c9087c09b695308a59dd9be388f9a0 Mon Sep 17 00:00:00 2001 From: raver119 Date: Wed, 13 May 2020 16:00:54 +0300 Subject: [PATCH 7/8] [WIP] Few more pre-release fixes (#461) * error code check in CudaMemoryManager Signed-off-by: raver119@gmail.com * clear Signed-off-by: raver119@gmail.com * clear model before exiting Signed-off-by: raver119@gmail.com * MultiLayerNetwork/ComputationGraph.close() [WIP] (#460) * MultiLayerNetwork/ComputationGraph.close() Signed-off-by: Alex Black * Copyright header Signed-off-by: Alex Black * Fix Signed-off-by: Alex Black * - fix for handling release of nested DataBuffers - couple of additional tests for released DataBuffers Signed-off-by: raver119@gmail.com * PW test: increase number of epochs slightly Signed-off-by: raver119@gmail.com Co-authored-by: Alex Black --- .../nn/misc/CloseNetworkTests.java | 151 ++++++++++++++++++ .../optimize/solver/TestOptimizers.java | 4 + .../deeplearning4j/plot/BarnesHutTsne.java | 5 + .../nn/modelimport/keras/Temp.java | 4 + .../java/org/deeplearning4j/nn/api/Model.java | 3 + .../nn/graph/ComputationGraph.java | 24 +++ .../nn/layers/AbstractLayer.java | 5 + .../layers/recurrent/BidirectionalLayer.java | 5 + .../variational/VariationalAutoencoder.java | 5 + .../nn/layers/wrapper/BaseWrapperLayer.java | 5 + .../nn/multilayer/MultiLayerNetwork.java | 23 +++ .../parallelism/trainer/DefaultTrainer.java | 8 + .../parallelism/ParallelWrapperTest.java | 2 +- .../linalg/api/buffer/BaseDataBuffer.java | 3 + .../nd4j/jita/memory/CudaMemoryManager.java | 8 +- .../jcublas/buffer/BaseCudaDataBuffer.java | 9 +- .../buffer/BaseCudaDataBufferTest.java | 23 +++ .../nativecpu/buffer/BaseCpuDataBuffer.java | 3 + 18 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java create mode 100644 deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/Temp.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java new file mode 100644 index 000000000..882ec8479 --- /dev/null +++ b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java @@ -0,0 +1,151 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.deeplearning4j.nn.misc; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.Updater; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; + +import static org.junit.Assert.assertTrue; + +public class CloseNetworkTests extends BaseDL4JTest { + + public static MultiLayerNetwork getTestNet() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Adam(1e-3)) + .list() + .layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(3, 3).activation(Activation.TANH).build()) + .layer(new BatchNormalization.Builder().nOut(5).build()) + .layer(new SubsamplingLayer.Builder().build()) + .layer(new DenseLayer.Builder().nOut(10).activation(Activation.RELU).build()) + .layer(new OutputLayer.Builder().nOut(10).build()) + .setInputType(InputType.convolutional(28, 28, 1)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + return net; + } + + @Test + public void testCloseMLN() { + for (boolean train : new boolean[]{false, true}) { + for (boolean test : new boolean[]{false, true}) { + MultiLayerNetwork net = getTestNet(); + + INDArray f = Nd4j.rand(DataType.FLOAT, 16, 1, 28, 28); + INDArray l = TestUtils.randomOneHot(16, 10); + + if (train) { + for (int i = 0; i < 3; i++) { + net.fit(f, l); + } + } + + if (test) { + for (int i = 0; i < 3; i++) { + net.output(f); + } + } + + net.close(); + + assertTrue(net.params().wasClosed()); + if(train) { + assertTrue(net.getGradientsViewArray().wasClosed()); + Updater u = net.getUpdater(false); + assertTrue(u.getStateViewArray().wasClosed()); + } + + //Make sure we don't get crashes etc when trying to use after closing + try { + net.output(f); + } catch (IllegalStateException e) { + String msg = e.getMessage(); + assertTrue(msg, msg.contains("released")); + } + + try { + net.fit(f, l); + } catch (IllegalStateException e) { + String msg = e.getMessage(); + assertTrue(msg, msg.contains("released")); + } + } + } + } + + @Test + public void testCloseCG() { + for (boolean train : new boolean[]{false, true}) { + for (boolean test : new boolean[]{false, true}) { + ComputationGraph net = getTestNet().toComputationGraph(); + + INDArray f = Nd4j.rand(DataType.FLOAT, 16, 1, 28, 28); + INDArray l = TestUtils.randomOneHot(16, 10); + + if (train) { + for (int i = 0; i < 3; i++) { + net.fit(new INDArray[]{f}, new INDArray[]{l}); + } + } + + if (test) { + for (int i = 0; i < 3; i++) { + net.output(f); + } + } + + net.close(); + + assertTrue(net.params().wasClosed()); + if(train) { + assertTrue(net.getGradientsViewArray().wasClosed()); + Updater u = net.getUpdater(false); + assertTrue(u.getStateViewArray().wasClosed()); + } + + //Make sure we don't get crashes etc when trying to use after closing + try { + net.output(f); + } catch (IllegalStateException e) { + String msg = e.getMessage(); + assertTrue(msg, msg.contains("released")); + } + + try { + net.fit(new INDArray[]{f}, new INDArray[]{l}); + } catch (IllegalStateException e) { + String msg = e.getMessage(); + assertTrue(msg, msg.contains("released")); + } + } + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java index 30cce7c28..f79763bfe 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java +++ b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java @@ -1035,5 +1035,9 @@ public class TestOptimizers extends BaseDL4JTest { public boolean updaterDivideByMinibatch(String paramName) { return true; } + + @Override + public void close(){ + } } } diff --git a/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java index a9f92cefc..8cd984044 100644 --- a/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java +++ b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java @@ -1055,4 +1055,9 @@ public class BarnesHutTsne implements Model { } + + @Override + public void close(){ + //No-op + } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/Temp.java b/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/Temp.java new file mode 100644 index 000000000..a93eb558c --- /dev/null +++ b/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/Temp.java @@ -0,0 +1,4 @@ +package org.deeplearning4j.nn.modelimport.keras; + +public class Temp { +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Model.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Model.java index 2d5314569..49b32dcc2 100755 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Model.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Model.java @@ -233,4 +233,7 @@ public interface Model { * Apply any constraints to the model */ void applyConstraints(int iteration, int epoch); + + + void close(); } diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java index 571afea7b..2f7bd45ee 100755 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java @@ -4824,4 +4824,28 @@ public class ComputationGraph implements Serializable, Model, NeuralNetwork { if (cg.getUpdater() != null && cg.getUpdater(false).getStateViewArray() != null) this.getUpdater(true).getStateViewArray().assign(cg.getUpdater(false).getStateViewArray()); } + + /** + * Close the network and deallocate all native memory, including: parameters, gradients, updater memory and workspaces + * Note that the network should not be used again for any purpose after it has been closed + */ + @Override + public void close(){ + //Close the INDArray and dealloc + if(flattenedParams.closeable()) + flattenedParams.close(); + + if(flattenedGradients != null && flattenedGradients.closeable()) + flattenedGradients.close(); + + Updater u = getUpdater(false); + if(u != null && u.getStateViewArray() != null) { + INDArray state = u.getStateViewArray(); + if(state.closeable()) + state.close(); + } + + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + System.gc(); + } } diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java index da03b19c5..f3d009a6c 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java @@ -428,4 +428,9 @@ public abstract class AbstractLayer Date: Wed, 13 May 2020 18:36:38 +0400 Subject: [PATCH 8/8] casting fixes (#464) --- .../keras/layers/advanced/activations/KerasReLU.java | 4 ++-- .../layers/normalization/KerasBatchNormalization.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java index 14c4b3d73..3a30ec9ef 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java +++ b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java @@ -63,10 +63,10 @@ public class KerasReLU extends KerasLayer { double negativeSlope = 0.0; double threshold = 0.0; if (innerConfig.containsKey("negative_slope")) { - negativeSlope = (double) innerConfig.get("negative_slope"); + negativeSlope = ((Number)innerConfig.get("negative_slope")).doubleValue(); } if (innerConfig.containsKey("threshold")) { - threshold = (double) innerConfig.get("threshold"); + threshold = ((Number)innerConfig.get("threshold")).doubleValue(); } this.layer = new ActivationLayer.Builder().name(this.layerName) diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java index 7f7d8dc4c..c27f753d1 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java +++ b/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java @@ -32,6 +32,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -351,6 +352,10 @@ public class KerasBatchNormalization extends KerasLayer { private int getBatchNormAxis(Map layerConfig) throws InvalidKerasConfigurationException { Map innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf); - return (int) innerConfig.get(LAYER_FIELD_AXIS); + Object batchNormAxis = innerConfig.get(LAYER_FIELD_AXIS); + if (batchNormAxis instanceof List){ + return ((Number)((List)batchNormAxis).get(0)).intValue(); + } + return ((Number)innerConfig.get(LAYER_FIELD_AXIS)).intValue(); } }