From 871073e4a4c5e01ba03df68624cae8f7837c5e50 Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 8 May 2023 09:22:38 +0200 Subject: [PATCH] Fixing tests Signed-off-by: brian --- ...TestSparkMultiLayerParameterAveraging.java | 12 ++-- .../src/test/java/net/brutex/gan/App.java | 62 ++++++++++--------- .../deallocation/DeallocatorService.java | 4 ++ .../earlystopping/TestEarlyStopping.java | 2 +- .../gradientcheck/AttentionLayerTest.java | 2 +- .../nn/conf/layers/LayerBuilderTest.java | 2 +- .../nn/conf/layers/LayerConfigTest.java | 32 +++++----- .../layers/LayerConfigValidationTest.java | 12 ++-- .../samediff/testlayers/SameDiffConv.java | 12 ---- .../samediff/testlayers/SameDiffDense.java | 8 --- .../testlayers/SameDiffMSEOutputLayer.java | 5 -- .../TransferLearningComplex.java | 2 +- .../TransferLearningMLNTest.java | 10 +-- .../nn/updater/TestUpdaters.java | 36 +++++------ .../nn/updater/custom/TestCustomUpdater.java | 16 ++--- .../regressiontest/RegressionTest050.java | 26 ++++---- .../regressiontest/RegressionTest060.java | 26 ++++---- .../regressiontest/RegressionTest071.java | 26 ++++---- .../regressiontest/RegressionTest080.java | 34 +++++----- .../regressiontest/RegressionTest100a.java | 8 +-- .../regressiontest/RegressionTest100b3.java | 12 ++-- .../regressiontest/RegressionTest100b4.java | 22 +++---- .../regressiontest/RegressionTest100b6.java | 22 +++---- .../customlayer100a/CustomLayer.java | 2 +- .../gradientcheck/GradientCheckUtil.java | 4 +- .../NeuralNetBaseBuilderConfiguration.java | 2 +- .../nn/conf/layers/ActivationLayer.java | 2 +- .../nn/conf/layers/AutoEncoder.java | 2 +- .../conf/layers/BaseLayerConfiguration.java | 13 +++- .../nn/conf/layers/BaseOutputLayer.java | 2 +- .../nn/conf/layers/ConvolutionLayer.java | 2 +- .../nn/conf/layers/DenseLayer.java | 4 +- .../nn/conf/layers/EmbeddingLayer.java | 2 +- .../conf/layers/EmbeddingSequenceLayer.java | 2 +- .../nn/conf/layers/LayerConfiguration.java | 30 ++++++--- .../nn/conf/layers/LocallyConnected1D.java | 11 ---- .../nn/conf/layers/LocallyConnected2D.java | 11 ---- .../nn/conf/layers/NoParamLayer.java | 3 +- .../nn/conf/layers/OutputLayer.java | 4 +- .../nn/conf/layers/PReLULayer.java | 2 +- .../conf/layers/RecurrentAttentionLayer.java | 13 ++-- .../misc/ElementWiseMultiplicationLayer.java | 2 +- .../layers/misc/FrozenLayerWithBackprop.java | 3 + .../samediff/AbstractSameDiffLayer.java | 43 +++++-------- .../variational/VariationalAutoencoder.java | 2 +- .../BaseWrapperLayerConfiguration.java | 12 ++-- .../conf/serde/BaseNetConfigDeserializer.java | 4 +- ...utationGraphConfigurationDeserializer.java | 2 +- .../NeuralNetConfigurationDeserializer.java | 2 +- .../nn/conf/weightnoise/WeightNoise.java | 1 + .../nn/layers/AbstractLayer.java | 3 +- .../nn/layers/pooling/GlobalPoolingLayer.java | 2 +- .../nn/layers/recurrent/LSTMHelpers.java | 2 +- .../nn/multilayer/MultiLayerNetwork.java | 5 +- .../nn/params/DefaultParamInitializer.java | 24 +++---- .../FineTuneConfiguration.java | 2 +- .../org/deeplearning4j/util/NetworkUtils.java | 8 +-- ...TestSparkMultiLayerParameterAveraging.java | 12 ++-- .../ops/executioner/CudaExecutioner.java | 9 +-- .../ui/module/train/TrainModule.java | 2 +- 60 files changed, 306 insertions(+), 338 deletions(-) diff --git a/.old/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java b/.old/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java index d8740af1a..90128a4cd 100644 --- a/.old/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java +++ b/.old/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java @@ -217,14 +217,14 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { MultiLayerNetwork netCopy = sparkNet.getNetwork().clone(); netCopy.fit(data); - IUpdater expectedUpdater = ((BaseLayer) netCopy.conf().getLayer()).getIUpdater(); - double expectedLR = ((Nesterovs)((BaseLayer) netCopy.conf().getLayer()).getIUpdater()).getLearningRate(); - double expectedMomentum = ((Nesterovs)((BaseLayer) netCopy.conf().getLayer()).getIUpdater()).getMomentum(); + IUpdater expectedUpdater = ((BaseLayer) netCopy.conf().getLayer()).getUpdater(); + double expectedLR = ((Nesterovs)((BaseLayer) netCopy.conf().getLayer()).getUpdater()).getLearningRate(); + double expectedMomentum = ((Nesterovs)((BaseLayer) netCopy.conf().getLayer()).getUpdater()).getMomentum(); - IUpdater actualUpdater = ((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getIUpdater(); + IUpdater actualUpdater = ((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getUpdater(); sparkNet.fit(sparkData); - double actualLR = ((Nesterovs)((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getIUpdater()).getLearningRate(); - double actualMomentum = ((Nesterovs)((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getIUpdater()).getMomentum(); + double actualLR = ((Nesterovs)((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getUpdater()).getLearningRate(); + double actualMomentum = ((Nesterovs)((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getUpdater()).getMomentum(); assertEquals(expectedUpdater, actualUpdater); assertEquals(expectedLR, actualLR, 0.01); diff --git a/brutex-extended-tests/src/test/java/net/brutex/gan/App.java b/brutex-extended-tests/src/test/java/net/brutex/gan/App.java index 18a0fc50d..5d06092ab 100644 --- a/brutex-extended-tests/src/test/java/net/brutex/gan/App.java +++ b/brutex-extended-tests/src/test/java/net/brutex/gan/App.java @@ -47,6 +47,7 @@ import org.datavec.image.transform.ShowImageTransform; import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; import org.deeplearning4j.nn.conf.GradientNormalization; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.Distribution; import org.deeplearning4j.nn.conf.distribution.NormalDistribution; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.ActivationLayer; @@ -77,10 +78,10 @@ public class App { private static final double LEARNING_RATE = 0.000002; private static final double GRADIENT_THRESHOLD = 100.0; - private static final int X_DIM = 28; - private static final int Y_DIM = 28; + private static final int X_DIM = 20 ; + private static final int Y_DIM = 20; private static final int CHANNELS = 1; - private static final int batchSize = 9; + private static final int batchSize = 10; private static final int INPUT = 128; private static final int OUTPUT_PER_PANEL = 4; @@ -97,12 +98,13 @@ public class App { return new LayerConfiguration[] { DenseLayer.builder().nIn(INPUT).nOut(X_DIM*Y_DIM*CHANNELS).weightInit(WeightInit.NORMAL).build(), ActivationLayer.builder(Activation.LEAKYRELU).build(), + DenseLayer.builder().nIn(X_DIM*Y_DIM*CHANNELS).nOut(X_DIM*Y_DIM).build(), ActivationLayer.builder(new ActivationLReLU(0.2)).build(), DenseLayer.builder().nIn(X_DIM*Y_DIM).nOut(X_DIM*Y_DIM).build(), ActivationLayer.builder(new ActivationLReLU(0.2)).build(), - DenseLayer.builder().nIn(X_DIM*Y_DIM).nOut(X_DIM*Y_DIM*CHANNELS).activation(Activation.TANH) - .build() + + DenseLayer.builder().nIn(X_DIM*Y_DIM*CHANNELS).nOut(X_DIM*Y_DIM*CHANNELS).activation(Activation.TANH).build() }; } @@ -131,36 +133,40 @@ public class App { private static LayerConfiguration[] disLayers() { return new LayerConfiguration[]{ - DenseLayer.builder().nOut(X_DIM*Y_DIM*CHANNELS*2).build(), //input is set by setInputType on the network + DenseLayer.builder().name("1.Dense").nOut(X_DIM*Y_DIM*CHANNELS).build(), //input is set by setInputType on the network ActivationLayer.builder(new ActivationLReLU(0.2)).build(), DropoutLayer.builder(1 - 0.5).build(), - DenseLayer.builder().nIn(X_DIM * Y_DIM*CHANNELS*2).nOut(X_DIM*Y_DIM*CHANNELS*4).build(), //HxBxC + DenseLayer.builder().name("2.Dense").nIn(X_DIM * Y_DIM*CHANNELS).nOut(X_DIM*Y_DIM*CHANNELS*4).build(), //HxBxC ActivationLayer.builder(new ActivationLReLU(0.2)).build(), DropoutLayer.builder(1 - 0.5).build(), - DenseLayer.builder().nIn(X_DIM*Y_DIM*CHANNELS*4).nOut(X_DIM*Y_DIM*CHANNELS).build(), + DenseLayer.builder().name("3.Dense").nIn(X_DIM*Y_DIM*CHANNELS*4).nOut(X_DIM*Y_DIM*CHANNELS).build(), ActivationLayer.builder(new ActivationLReLU(0.2)).build(), DropoutLayer.builder(1 - 0.5).build(), - DenseLayer.builder().nIn(X_DIM*Y_DIM*CHANNELS).nOut(X_DIM*Y_DIM).build(), + DenseLayer.builder().name("4.Dense").nIn(X_DIM*Y_DIM*CHANNELS).nOut(X_DIM*Y_DIM).build(), ActivationLayer.builder(new ActivationLReLU(0.2)).build(), DropoutLayer.builder(1 - 0.5).build(), - OutputLayer.builder().lossFunction(LossFunction.XENT).nIn(X_DIM*Y_DIM).nOut(1).activation(Activation.SIGMOID).build() + + OutputLayer.builder().name("dis-output").lossFunction(LossFunction.XENT).nIn(X_DIM*Y_DIM).nOut(1).activation(Activation.SIGMOID).build() }; } private static NeuralNetConfiguration discriminator() { - NeuralNetConfiguration conf = NeuralNetConfiguration.builder() - .seed(42) - .updater(UPDATER) - .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) - .gradientNormalizationThreshold(GRADIENT_THRESHOLD) - .weightInit(WeightInit.XAVIER) - //.weightInitFn(new WeightInitXavier()) - //.activationFn(new ActivationIdentity()) - .activation(Activation.IDENTITY) - .layersFromArray(disLayers()) - .inputType(InputType.convolutional(X_DIM, Y_DIM, CHANNELS)) - .build(); + NeuralNetConfiguration conf = + NeuralNetConfiguration.builder() + .seed(42) + .updater(UPDATER) + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) + .gradientNormalizationThreshold(GRADIENT_THRESHOLD) + .weightInit(WeightInit.XAVIER) + //.weightNoise(new WeightNoise(new NormalDistribution(0.5, 0.5))) + .weightNoise(null) + // .weightInitFn(new WeightInitXavier()) + // .activationFn(new ActivationIdentity()) + .activation(Activation.IDENTITY) + .layersFromArray(disLayers()) + .inputType(InputType.convolutional(X_DIM, Y_DIM, CHANNELS)) + .build(); ((NeuralNetConfiguration) conf).init(); return conf; @@ -171,7 +177,7 @@ public class App { LayerConfiguration[] disLayers = Arrays.stream(disLayers()) .map((layer) -> { if (layer instanceof DenseLayer || layer instanceof OutputLayer) { - return FrozenLayerWithBackprop.builder(layer); + return FrozenLayerWithBackprop.builder(layer).name("frozen-for-"+layer.getName()).build(); } else { return layer; } @@ -204,7 +210,7 @@ public class App { public static void main(String... args) throws Exception { log.info("\u001B[32m Some \u001B[1m green \u001B[22m text \u001B[0m \u001B[7m Inverted\u001B[0m "); - Nd4j.getMemoryManager().setAutoGcWindow(15 * 1000); + Nd4j.getMemoryManager().setAutoGcWindow(500); // MnistDataSetIterator trainData = new MnistDataSetIterator(128, true, 45); // FileSplit fileSplit = new FileSplit(new File("c:/users/brian/downloads/flowers"), NativeImageLoader.getALLOWED_FORMATS()); @@ -236,10 +242,10 @@ public class App { copyParams(gen, dis, gan); - gen.addTrainingListeners(new PerformanceListener(10, true)); - dis.addTrainingListeners(new PerformanceListener(10, true)); - gan.addTrainingListeners(new PerformanceListener(10, true)); - gan.addTrainingListeners(new ScoreToChartListener("gan")); + gen.addTrainingListeners(new PerformanceListener(15, true)); + //dis.addTrainingListeners(new PerformanceListener(10, true)); + //gan.addTrainingListeners(new PerformanceListener(10, true)); + //gan.addTrainingListeners(new ScoreToChartListener("gan")); //dis.setListeners(new ScoreToChartListener("dis")); System.out.println(gan.toString()); diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java index 5934a6d14..b19a66d50 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java @@ -107,6 +107,9 @@ public class DeallocatorService { boolean canRun = true; long cnt = 0; while (canRun) { + log.trace("Starting deallocator threat with name '{}'. isPeriodicGc: {}, AutoGcWindow: {}. Current allocated memory: {}" + ,this.getName(), Nd4j.getMemoryManager().isPeriodicGcActive() + , Nd4j.getMemoryManager().getAutoGcWindow(), Nd4j.getMemoryManager().allocatedMemory(deviceId)); // if periodicGc is enabled, only first thread will call for it if (Nd4j.getMemoryManager().isPeriodicGcActive() && threadIdx == 0 && Nd4j.getMemoryManager().getAutoGcWindow() > 0) { val reference = (DeallocatableReference) queue.poll(); @@ -120,6 +123,7 @@ public class DeallocatorService { } } else { // invoking deallocator + log.trace("Deallocate reference {}", reference.getId()); reference.getDeallocator().deallocate(); referenceMap.remove(reference.getId()); } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java index a4923e5da..45d179da1 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java @@ -498,7 +498,7 @@ public class TestEarlyStopping extends BaseDL4JTest { assertEquals(net.getNetConfiguration().getOptimizationAlgo(), mln.getNetConfiguration().getOptimizationAlgo()); BaseLayerConfiguration bl = (BaseLayerConfiguration) net.getLayerConfiguration(); assertEquals(bl.getActivationFn().toString(), ((BaseLayerConfiguration) mln.getLayerConfiguration()).getActivationFn().toString()); - assertEquals(bl.getIUpdater(), ((BaseLayerConfiguration) mln.getLayerConfiguration()).getIUpdater()); + assertEquals(bl.getUpdater(), ((BaseLayerConfiguration) mln.getLayerConfiguration()).getUpdater()); } @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java index 286f93992..13a968a2e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java @@ -306,7 +306,7 @@ public class AttentionLayerTest extends BaseDL4JTest { .activation(Activation.IDENTITY) .updater(new NoOp()) .weightInit(WeightInit.XAVIER) - .list() + .layer(LSTM.builder().nOut(layerSize).build()) .layer(RecurrentAttentionLayer.builder().nIn(layerSize).nOut(layerSize).nHeads(1).projectInput(false).hasBias(false).build()) .layer(GlobalPoolingLayer.builder().poolingType(PoolingType.AVG).build()) diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java index b8e8909e5..cdd6e0167 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java @@ -76,7 +76,7 @@ public class LayerBuilderTest extends BaseDL4JTest { assertEquals(act, layer.getActivationFn()); assertEquals(weight.getWeightInitFunction(), layer.getWeightInit()); assertEquals(new Dropout(dropOut), layer.getDropOut()); - assertEquals(updater, layer.getIUpdater()); + assertEquals(updater, layer.getUpdater()); assertEquals(gradNorm, layer.getGradientNormalization()); assertEquals(gradNormThreshold, layer.getGradientNormalizationThreshold(), 0.0); } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java index bce68f0c9..56240b3cf 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java @@ -213,8 +213,8 @@ public class LayerConfigTest extends BaseDL4JTest { MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); - assertEquals(0.1, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); - assertEquals(0.1, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + assertEquals(0.1, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + assertEquals(0.1, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); Map testMomentumAfter2 = new HashMap<>(); testMomentumAfter2.put(0, 0.2); @@ -227,8 +227,8 @@ public class LayerConfigTest extends BaseDL4JTest { net = new MultiLayerNetwork(conf); net.init(); - assertEquals(0.1, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); - assertEquals(0.2, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + assertEquals(0.1, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + assertEquals(0.2, ((Nesterovs)((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); } @Test @@ -239,10 +239,10 @@ public class LayerConfigTest extends BaseDL4JTest { MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); - assertTrue(((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater() instanceof AdaDelta); - assertTrue(((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater() instanceof AdaDelta); - assertEquals(0.5, ((AdaDelta)((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater()).getRho(), 0.0); - assertEquals(0.01, ((AdaDelta)((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater()).getRho(), 0.0); + assertTrue(((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater() instanceof AdaDelta); + assertTrue(((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater() instanceof AdaDelta); + assertEquals(0.5, ((AdaDelta)((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater()).getRho(), 0.0); + assertEquals(0.01, ((AdaDelta)((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater()).getRho(), 0.0); conf = NeuralNetConfiguration.builder().updater(new RmsProp(1.0, 2.0, RmsProp.DEFAULT_RMSPROP_EPSILON)) .layer(0, DenseLayer.builder().nIn(2).nOut(2).updater(new RmsProp(1.0, 1.0, RmsProp.DEFAULT_RMSPROP_EPSILON)).build()) @@ -252,10 +252,10 @@ public class LayerConfigTest extends BaseDL4JTest { net = new MultiLayerNetwork(conf); net.init(); - assertTrue(((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater() instanceof RmsProp); - assertTrue(((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater() instanceof AdaDelta); - assertEquals(1.0, ((RmsProp) ((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater()).getRmsDecay(), 0.0); - assertEquals(0.5, ((AdaDelta) ((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater()).getRho(), 0.0); + assertTrue(((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater() instanceof RmsProp); + assertTrue(((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater() instanceof AdaDelta); + assertEquals(1.0, ((RmsProp) ((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater()).getRmsDecay(), 0.0); + assertEquals(0.5, ((AdaDelta) ((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater()).getRho(), 0.0); } @@ -270,10 +270,10 @@ public class LayerConfigTest extends BaseDL4JTest { MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); - assertEquals(0.5, ((Adam) ((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater()).getBeta1(), 0.0); - assertEquals(0.6, ((Adam) ((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater()).getBeta1(), 0.0); - assertEquals(0.5, ((Adam) ((BaseLayerConfiguration) conf.getConf(0).getLayer()).getIUpdater()).getBeta2(), 0.0); - assertEquals(0.7, ((Adam) ((BaseLayerConfiguration) conf.getConf(1).getLayer()).getIUpdater()).getBeta2(), 0.0); + assertEquals(0.5, ((Adam) ((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater()).getBeta1(), 0.0); + assertEquals(0.6, ((Adam) ((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater()).getBeta1(), 0.0); + assertEquals(0.5, ((Adam) ((BaseLayerConfiguration) conf.getConf(0).getLayer()).getUpdater()).getBeta2(), 0.0); + assertEquals(0.7, ((Adam) ((BaseLayerConfiguration) conf.getConf(1).getLayer()).getUpdater()).getBeta2(), 0.0); } @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java index f15342535..718e17cd7 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java @@ -163,12 +163,12 @@ public class LayerConfigValidationTest extends BaseDL4JTest { net.init(); BaseLayerConfiguration layerConf = (BaseLayerConfiguration) net.getLayer(0).getLayerConfiguration(); - assertEquals(expectedMomentum, ((Nesterovs) layerConf.getIUpdater()).getMomentum(), 1e-3); + assertEquals(expectedMomentum, ((Nesterovs) layerConf.getUpdater()).getMomentum(), 1e-3); assertNull(TestUtils.getL1Reg(layerConf.getRegularization())); assertEquals(0.5, TestUtils.getL2(layerConf), 1e-3); BaseLayerConfiguration layerConf1 = (BaseLayerConfiguration) net.getLayer(1).getLayerConfiguration(); - assertEquals(0.4, ((Nesterovs) layerConf1.getIUpdater()).getMomentum(), 1e-3); + assertEquals(0.4, ((Nesterovs) layerConf1.getUpdater()).getMomentum(), 1e-3); // Adam Updater conf = NeuralNetConfiguration.builder().updater(new Adam(0.3)) @@ -183,8 +183,8 @@ public class LayerConfigValidationTest extends BaseDL4JTest { assertEquals(0.5, TestUtils.getL2(layerConf), 1e-3); layerConf1 = (BaseLayerConfiguration) net.getLayer(1).getLayerConfiguration(); - assertEquals(expectedAdamMeanDecay, ((Adam) layerConf1.getIUpdater()).getBeta1(), 1e-3); - assertEquals(expectedAdamVarDecay, ((Adam) layerConf1.getIUpdater()).getBeta2(), 1e-3); + assertEquals(expectedAdamMeanDecay, ((Adam) layerConf1.getUpdater()).getBeta1(), 1e-3); + assertEquals(expectedAdamVarDecay, ((Adam) layerConf1.getUpdater()).getBeta2(), 1e-3); assertEquals(new WeightInitDistribution(expectedDist), layerConf1.getWeightInit()); assertNull(TestUtils.getL1Reg(layerConf1.getRegularization())); assertNull(TestUtils.getL2Reg(layerConf1.getRegularization())); @@ -197,12 +197,12 @@ public class LayerConfigValidationTest extends BaseDL4JTest { net.init(); layerConf = (BaseLayerConfiguration) net.getLayer(0).getLayerConfiguration(); - assertEquals(expectedRmsDecay, ((RmsProp) layerConf.getIUpdater()).getRmsDecay(), 1e-3); + assertEquals(expectedRmsDecay, ((RmsProp) layerConf.getUpdater()).getRmsDecay(), 1e-3); assertNull(TestUtils.getL1Reg(layerConf.getRegularization())); assertNull(TestUtils.getL2Reg(layerConf.getRegularization())); layerConf1 = (BaseLayerConfiguration) net.getLayer(1).getLayerConfiguration(); - assertEquals(0.4, ((RmsProp) layerConf1.getIUpdater()).getRmsDecay(), 1e-3); + assertEquals(0.4, ((RmsProp) layerConf1.getUpdater()).getRmsDecay(), 1e-3); } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java index 35f87c35a..5125829a7 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java @@ -164,16 +164,4 @@ public class SameDiffConv extends SameDiffLayer { return activation.asSameDiff("out", sameDiff, conv); } - @Override - public void applyGlobalConfigToLayer(NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) { - NeuralNetConfiguration clone = globalConfig.clone().build(); - if (activation == null) { - activation = SameDiffLayerUtils.fromIActivation(clone.getActivation()); - } - if (convolutionMode == null) { - convolutionMode = clone.getConvolutionMode(); - } - } - - } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java index 7f3e41915..db414ee12 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java @@ -114,14 +114,6 @@ public class SameDiffDense extends SameDiffLayer { return activation.asSameDiff("out", sd, z); } - @Override - public void applyGlobalConfigToLayer( - NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) { - NeuralNetConfiguration clone = globalConfig.clone().build(); - if (activation == null) { - activation = SameDiffLayerUtils.fromIActivation(clone.getActivation()); - } - } public char paramReshapeOrder(String param) { // To match DL4J for easy comparison diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java index 24a2b01cd..a90e09e62 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java @@ -21,7 +21,6 @@ package org.deeplearning4j.nn.layers.samediff.testlayers; import java.util.Map; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.samediff.SDLayerParams; import org.deeplearning4j.nn.conf.layers.samediff.SameDiffOutputLayer; @@ -87,8 +86,4 @@ public class SameDiffMSEOutputLayer extends SameDiffOutputLayer { // To match DL4J for easy comparison return 'f'; } - - @Override - public void applyGlobalConfigToLayer( - NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) {} } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java index 17d711726..e29a90a7e 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java @@ -99,7 +99,7 @@ public class TransferLearningComplex extends BaseDL4JTest { //Also check config: BaseLayerConfiguration bl = ((BaseLayerConfiguration) l.getLayerConfiguration()); - assertEquals(new Adam(2e-2), bl.getIUpdater()); + assertEquals(new Adam(2e-2), bl.getUpdater()); assertEquals(Activation.LEAKYRELU.getActivationFunction(), bl.getActivationFn()); } assertTrue(cFound); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java index 1776d0d70..73f535be7 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java @@ -92,7 +92,7 @@ public class TransferLearningMLNTest extends BaseDL4JTest { for (org.deeplearning4j.nn.api.Layer l : modelNow.getLayers()) { BaseLayerConfiguration bl = ((BaseLayerConfiguration) l.getLayerConfiguration()); - assertEquals(new RmsProp(0.5), bl.getIUpdater()); + assertEquals(new RmsProp(0.5), bl.getUpdater()); } @@ -504,13 +504,13 @@ public class TransferLearningMLNTest extends BaseDL4JTest { //Check original net isn't modified: BaseLayerConfiguration l0 = (BaseLayerConfiguration) net.getLayer(0).getLayerConfiguration(); - assertEquals(new Adam(1e-4), l0.getIUpdater()); + assertEquals(new Adam(1e-4), l0.getUpdater()); assertEquals(Activation.TANH.getActivationFunction(), l0.getActivationFn()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); assertEquals(0.1, TestUtils.getL1(l0), 1e-6); BaseLayerConfiguration l1 = (BaseLayerConfiguration) net.getLayer(1).getLayerConfiguration(); - assertEquals(new Adam(1e-4), l1.getIUpdater()); + assertEquals(new Adam(1e-4), l1.getUpdater()); assertEquals(Activation.HARDSIGMOID.getActivationFunction(), l1.getActivationFn()); assertEquals(new WeightInitRelu(), l1.getWeightInit()); assertEquals(0.2, TestUtils.getL2(l1), 1e-6); @@ -519,13 +519,13 @@ public class TransferLearningMLNTest extends BaseDL4JTest { //Check new net has only the appropriate things modified (i.e., LR) l0 = (BaseLayerConfiguration) net2.getLayer(0).getLayerConfiguration(); - assertEquals(new Adam(2e-2), l0.getIUpdater()); + assertEquals(new Adam(2e-2), l0.getUpdater()); assertEquals(Activation.TANH.getActivationFunction(), l0.getActivationFn()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); assertEquals(0.1, TestUtils.getL1(l0), 1e-6); l1 = (BaseLayerConfiguration) net2.getLayer(1).getLayerConfiguration(); - assertEquals(new Adam(2e-2), l1.getIUpdater()); + assertEquals(new Adam(2e-2), l1.getUpdater()); assertEquals(Activation.HARDSIGMOID.getActivationFunction(), l1.getActivationFn()); assertEquals(new WeightInitRelu(), l1.getWeightInit()); assertEquals(0.2, TestUtils.getL2(l1), 1e-6); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java index 0e2a77eff..17f06a0a3 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java @@ -100,7 +100,7 @@ public class TestUpdaters extends BaseDL4JTest { BaseLayer layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -145,7 +145,7 @@ public class TestUpdaters extends BaseDL4JTest { msdx.put(key, msdxTmp); count++; } - assertEquals(rho, ((AdaDelta)layer.getTypedLayerConfiguration().getIUpdater()).getRho(), 1e-4); + assertEquals(rho, ((AdaDelta)layer.getTypedLayerConfiguration().getUpdater()).getRho(), 1e-4); } assertEquals(4, count); @@ -166,7 +166,7 @@ public class TestUpdaters extends BaseDL4JTest { BaseLayer layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -186,7 +186,7 @@ public class TestUpdaters extends BaseDL4JTest { assertEquals(gradExpected, gradient.getGradientFor(entry.getKey())); count++; } - assertEquals(lr, ((AdaGrad)layer.getTypedLayerConfiguration().getIUpdater()).getLearningRate(), 1e-4); + assertEquals(lr, ((AdaGrad)layer.getTypedLayerConfiguration().getUpdater()).getLearningRate(), 1e-4); assertEquals(2, count); } @@ -210,7 +210,7 @@ public class TestUpdaters extends BaseDL4JTest { BaseLayer layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -246,8 +246,8 @@ public class TestUpdaters extends BaseDL4JTest { count++; } - assertEquals(beta1, ((Adam)layer.getTypedLayerConfiguration().getIUpdater()).getBeta1(), 1e-4); - assertEquals(beta2, ((Adam)layer.getTypedLayerConfiguration().getIUpdater()).getBeta2(), 1e-4); + assertEquals(beta1, ((Adam)layer.getTypedLayerConfiguration().getUpdater()).getBeta1(), 1e-4); + assertEquals(beta2, ((Adam)layer.getTypedLayerConfiguration().getUpdater()).getBeta2(), 1e-4); assertEquals(2, count); } @@ -274,7 +274,7 @@ public class TestUpdaters extends BaseDL4JTest { layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -363,7 +363,7 @@ public class TestUpdaters extends BaseDL4JTest { BaseLayer layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -399,8 +399,8 @@ public class TestUpdaters extends BaseDL4JTest { count++; } - assertEquals(beta1, ((AdaMax)layer.getTypedLayerConfiguration().getIUpdater()).getBeta1(), 1e-4); - assertEquals(beta2, ((AdaMax)layer.getTypedLayerConfiguration().getIUpdater()).getBeta2(), 1e-4); + assertEquals(beta1, ((AdaMax)layer.getTypedLayerConfiguration().getUpdater()).getBeta1(), 1e-4); + assertEquals(beta2, ((AdaMax)layer.getTypedLayerConfiguration().getUpdater()).getBeta2(), 1e-4); assertEquals(2, count); } @@ -419,7 +419,7 @@ public class TestUpdaters extends BaseDL4JTest { BaseLayer layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -444,7 +444,7 @@ public class TestUpdaters extends BaseDL4JTest { count++; } - assertEquals(mu, ((Nesterovs)layer.getTypedLayerConfiguration().getIUpdater()).getMomentum(), 1e-4); + assertEquals(mu, ((Nesterovs)layer.getTypedLayerConfiguration().getUpdater()).getMomentum(), 1e-4); assertEquals(2, count); } @@ -466,7 +466,7 @@ public class TestUpdaters extends BaseDL4JTest { BaseLayer layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); Updater updater = UpdaterCreator.getUpdater(layer); - int updaterStateSize = (int) layer.getTypedLayerConfiguration().getIUpdater().stateSize(numParams); + int updaterStateSize = (int) layer.getTypedLayerConfiguration().getUpdater().stateSize(numParams); INDArray updaterState = Nd4j.create(1, updaterStateSize); updater.setStateViewArray(layer, updaterState, true); @@ -496,7 +496,7 @@ public class TestUpdaters extends BaseDL4JTest { assertEquals(gradExpected, gradient.getGradientFor(entry.getKey())); lastG.put(key, lastGTmp); } - assertEquals(rmsDecay, ((RmsProp)layer.getTypedLayerConfiguration().getIUpdater()).getRmsDecay(), 1e-4); + assertEquals(rmsDecay, ((RmsProp)layer.getTypedLayerConfiguration().getUpdater()).getRmsDecay(), 1e-4); } @Test @@ -528,7 +528,7 @@ public class TestUpdaters extends BaseDL4JTest { gradExpected = val.mul(lr); assertEquals(gradExpected, gradient.getGradientFor(entry.getKey())); } - assertEquals(lr, ((Sgd)layer.getTypedLayerConfiguration().getIUpdater()).getLearningRate(), 1e-4); + assertEquals(lr, ((Sgd)layer.getTypedLayerConfiguration().getUpdater()).getLearningRate(), 1e-4); } @@ -770,7 +770,7 @@ public class TestUpdaters extends BaseDL4JTest { gradExpected = val.mul(lr); assertEquals(gradExpected, gradient.getGradientFor(entry.getKey())); } - assertEquals(lr, ((Sgd)layer.getTypedLayerConfiguration().getIUpdater()).getLearningRate(), 1e-4); + assertEquals(lr, ((Sgd)layer.getTypedLayerConfiguration().getUpdater()).getLearningRate(), 1e-4); //Test with pretrain == false @@ -798,7 +798,7 @@ public class TestUpdaters extends BaseDL4JTest { layer = (BaseLayer) conf.getFirstLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(gradients); updater = UpdaterCreator.getUpdater(layer); - assertEquals(lr, ((Sgd)layer.getTypedLayerConfiguration().getIUpdater()).getLearningRate(), 1e-4); + assertEquals(lr, ((Sgd)layer.getTypedLayerConfiguration().getUpdater()).getLearningRate(), 1e-4); } @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java index c909187fc..c7535ecaf 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java @@ -61,18 +61,18 @@ public class TestCustomUpdater extends BaseDL4JTest { .build(); //First: Check updater config - assertTrue(((BaseLayerConfiguration) conf1.getConf(0).getLayer()).getIUpdater() instanceof CustomIUpdater); - assertTrue(((BaseLayerConfiguration) conf1.getConf(1).getLayer()).getIUpdater() instanceof CustomIUpdater); - assertTrue(((BaseLayerConfiguration) conf2.getConf(0).getLayer()).getIUpdater() instanceof Sgd); - assertTrue(((BaseLayerConfiguration) conf2.getConf(1).getLayer()).getIUpdater() instanceof Sgd); + assertTrue(((BaseLayerConfiguration) conf1.getConf(0).getLayer()).getUpdater() instanceof CustomIUpdater); + assertTrue(((BaseLayerConfiguration) conf1.getConf(1).getLayer()).getUpdater() instanceof CustomIUpdater); + assertTrue(((BaseLayerConfiguration) conf2.getConf(0).getLayer()).getUpdater() instanceof Sgd); + assertTrue(((BaseLayerConfiguration) conf2.getConf(1).getLayer()).getUpdater() instanceof Sgd); - CustomIUpdater u0_0 = (CustomIUpdater) ((BaseLayerConfiguration) conf1.getConf(0).getLayer()).getIUpdater(); - CustomIUpdater u0_1 = (CustomIUpdater) ((BaseLayerConfiguration) conf1.getConf(1).getLayer()).getIUpdater(); + CustomIUpdater u0_0 = (CustomIUpdater) ((BaseLayerConfiguration) conf1.getConf(0).getLayer()).getUpdater(); + CustomIUpdater u0_1 = (CustomIUpdater) ((BaseLayerConfiguration) conf1.getConf(1).getLayer()).getUpdater(); assertEquals(lr, u0_0.getLearningRate(), 1e-6); assertEquals(lr, u0_1.getLearningRate(), 1e-6); - Sgd u1_0 = (Sgd) ((BaseLayerConfiguration) conf2.getConf(0).getLayer()).getIUpdater(); - Sgd u1_1 = (Sgd) ((BaseLayerConfiguration) conf2.getConf(1).getLayer()).getIUpdater(); + Sgd u1_0 = (Sgd) ((BaseLayerConfiguration) conf2.getConf(0).getLayer()).getUpdater(); + Sgd u1_1 = (Sgd) ((BaseLayerConfiguration) conf2.getConf(1).getLayer()).getUpdater(); assertEquals(lr, u1_0.getLearningRate(), 1e-6); assertEquals(lr, u1_1.getLearningRate(), 1e-6); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java index 14d04ac82..f7537ca3d 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java @@ -73,8 +73,8 @@ public class RegressionTest050 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); - assertEquals(new Nesterovs(0.15, 0.9), l0.getIUpdater()); - assertEquals(0.15, ((Nesterovs)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new Nesterovs(0.15, 0.9), l0.getUpdater()); + assertEquals(0.15, ((Nesterovs)l0.getUpdater()).getLearningRate(), 1e-6); OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer(); assertEquals("softmax", l1.getActivationFn().toString()); @@ -82,9 +82,9 @@ public class RegressionTest050 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); - assertEquals(new Nesterovs(0.15, 0.9), l1.getIUpdater()); - assertEquals(0.9, ((Nesterovs)l1.getIUpdater()).getMomentum(), 1e-6); - assertEquals(0.15, ((Nesterovs)l1.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new Nesterovs(0.15, 0.9), l1.getUpdater()); + assertEquals(0.9, ((Nesterovs)l1.getUpdater()).getMomentum(), 1e-6); + assertEquals(0.15, ((Nesterovs)l1.getUpdater()).getLearningRate(), 1e-6); int numParams = (int)net.numParams(); assertEquals(Nd4j.linspace(1, numParams, numParams, Nd4j.dataType()).reshape(1,numParams), net.getModelParams()); @@ -107,8 +107,8 @@ public class RegressionTest050 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l0.getDropOut()); assertEquals(0.1, TestUtils.getL1(l0), 1e-6); assertEquals(new WeightDecay(0.2, false), TestUtils.getWeightDecayReg(l0)); @@ -119,8 +119,8 @@ public class RegressionTest050 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getIUpdater()); - assertEquals(0.15, ((RmsProp)l1.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getUpdater()); + assertEquals(0.15, ((RmsProp)l1.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l1.getDropOut()); assertEquals(0.1, TestUtils.getL1(l1), 1e-6); assertEquals(new WeightDecay(0.2, false), TestUtils.getWeightDecayReg(l1)); @@ -146,8 +146,8 @@ public class RegressionTest050 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(3, l0.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertArrayEquals(new int[] {2, 2}, l0.getKernelSize()); assertArrayEquals(new int[] {1, 1}, l0.getStride()); assertArrayEquals(new int[] {0, 0}, l0.getPadding()); @@ -166,8 +166,8 @@ public class RegressionTest050 extends BaseDL4JTest { assertEquals(26 * 26 * 3, l2.getNIn()); assertEquals(5, l2.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); int numParams = (int)net.numParams(); assertEquals(Nd4j.linspace(1, numParams, numParams, Nd4j.dataType()).reshape(1,numParams), net.getModelParams()); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java index f4c46e7ee..66999a9e4 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java @@ -75,8 +75,8 @@ public class RegressionTest060 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); - assertEquals(new Nesterovs(0.15, 0.9), l0.getIUpdater()); - assertEquals(0.15, ((Nesterovs)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new Nesterovs(0.15, 0.9), l0.getUpdater()); + assertEquals(0.15, ((Nesterovs)l0.getUpdater()).getLearningRate(), 1e-6); OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer(); assertEquals("softmax", l1.getActivationFn().toString()); @@ -84,9 +84,9 @@ public class RegressionTest060 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); - assertEquals(new Nesterovs(0.15, 0.9), l1.getIUpdater()); - assertEquals(0.9, ((Nesterovs)l1.getIUpdater()).getMomentum(), 1e-6); - assertEquals(0.15, ((Nesterovs)l1.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new Nesterovs(0.15, 0.9), l1.getUpdater()); + assertEquals(0.9, ((Nesterovs)l1.getUpdater()).getMomentum(), 1e-6); + assertEquals(0.15, ((Nesterovs)l1.getUpdater()).getLearningRate(), 1e-6); int numParams = (int)net.numParams(); assertEquals(Nd4j.linspace(1, numParams, numParams, Nd4j.dataType()).reshape(1,numParams), net.getModelParams()); @@ -109,8 +109,8 @@ public class RegressionTest060 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l0.getDropOut()); assertEquals(0.1, TestUtils.getL1(l0), 1e-6); assertEquals(new WeightDecay(0.2, false), TestUtils.getWeightDecayReg(l0)); @@ -123,8 +123,8 @@ public class RegressionTest060 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getIUpdater()); - assertEquals(0.15, ((RmsProp)l1.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getUpdater()); + assertEquals(0.15, ((RmsProp)l1.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l1.getDropOut()); assertEquals(0.1, TestUtils.getL1(l1), 1e-6); assertEquals(new WeightDecay(0.2,false), TestUtils.getWeightDecayReg(l1)); @@ -152,8 +152,8 @@ public class RegressionTest060 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(3, l0.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertArrayEquals(new int[] {2, 2}, l0.getKernelSize()); assertArrayEquals(new int[] {1, 1}, l0.getStride()); assertArrayEquals(new int[] {0, 0}, l0.getPadding()); @@ -172,8 +172,8 @@ public class RegressionTest060 extends BaseDL4JTest { assertEquals(26 * 26 * 3, l2.getNIn()); assertEquals(5, l2.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java index f43c1e9ad..63fb200d5 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java @@ -76,8 +76,8 @@ public class RegressionTest071 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); - assertEquals(new Nesterovs(0.15, 0.9), l0.getIUpdater()); - assertEquals(0.15, ((Nesterovs)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new Nesterovs(0.15, 0.9), l0.getUpdater()); + assertEquals(0.15, ((Nesterovs)l0.getUpdater()).getLearningRate(), 1e-6); OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer(); assertEquals("softmax", l1.getActivationFn().toString()); @@ -85,9 +85,9 @@ public class RegressionTest071 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); - assertEquals(0.9, ((Nesterovs)l1.getIUpdater()).getMomentum(), 1e-6); - assertEquals(0.9, ((Nesterovs)l1.getIUpdater()).getMomentum(), 1e-6); - assertEquals(0.15, ((Nesterovs)l1.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(0.9, ((Nesterovs)l1.getUpdater()).getMomentum(), 1e-6); + assertEquals(0.9, ((Nesterovs)l1.getUpdater()).getMomentum(), 1e-6); + assertEquals(0.15, ((Nesterovs)l1.getUpdater()).getLearningRate(), 1e-6); long numParams = (int)net.numParams(); assertEquals(Nd4j.linspace(1, numParams, numParams).reshape(1,numParams), net.getModelParams()); @@ -110,8 +110,8 @@ public class RegressionTest071 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l0.getDropOut()); assertEquals(0.1, TestUtils.getL1(l0), 1e-6); assertEquals(new WeightDecay(0.2,false), TestUtils.getWeightDecayReg(l0)); @@ -124,8 +124,8 @@ public class RegressionTest071 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l1.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l1.getDropOut()); assertEquals(0.1, TestUtils.getL1(l1), 1e-6); assertEquals(new WeightDecay(0.2,false), TestUtils.getWeightDecayReg(l1)); @@ -153,8 +153,8 @@ public class RegressionTest071 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(3, l0.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertArrayEquals(new int[] {2, 2}, l0.getKernelSize()); assertArrayEquals(new int[] {1, 1}, l0.getStride()); assertArrayEquals(new int[] {0, 0}, l0.getPadding()); @@ -173,8 +173,8 @@ public class RegressionTest071 extends BaseDL4JTest { assertEquals(26 * 26 * 3, l2.getNIn()); assertEquals(5, l2.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getIUpdater()); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(new RmsProp(0.15, 0.96, RmsProp.DEFAULT_RMSPROP_EPSILON), l0.getUpdater()); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java index 379dd289a..992942048 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java @@ -75,10 +75,10 @@ public class RegressionTest080 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); - assertTrue(l0.getIUpdater() instanceof Nesterovs); - Nesterovs n = (Nesterovs) l0.getIUpdater(); + assertTrue(l0.getUpdater() instanceof Nesterovs); + Nesterovs n = (Nesterovs) l0.getUpdater(); assertEquals(0.9, n.getMomentum(), 1e-6); - assertEquals(0.15, ((Nesterovs)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(0.15, ((Nesterovs)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(0.15, n.getLearningRate(), 1e-6); @@ -88,9 +88,9 @@ public class RegressionTest080 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); - assertTrue(l1.getIUpdater() instanceof Nesterovs); - assertEquals(0.9, ((Nesterovs)l1.getIUpdater()).getMomentum(), 1e-6); - assertEquals(0.15, ((Nesterovs)l1.getIUpdater()).getLearningRate(), 1e-6); + assertTrue(l1.getUpdater() instanceof Nesterovs); + assertEquals(0.9, ((Nesterovs)l1.getUpdater()).getMomentum(), 1e-6); + assertEquals(0.15, ((Nesterovs)l1.getUpdater()).getLearningRate(), 1e-6); assertEquals(0.15, n.getLearningRate(), 1e-6); int numParams = (int)net.numParams(); @@ -114,11 +114,11 @@ public class RegressionTest080 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(4, l0.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l0.getWeightInit()); - assertTrue(l0.getIUpdater() instanceof RmsProp); - RmsProp r = (RmsProp) l0.getIUpdater(); + assertTrue(l0.getUpdater() instanceof RmsProp); + RmsProp r = (RmsProp) l0.getUpdater(); assertEquals(0.96, r.getRmsDecay(), 1e-6); assertEquals(0.15, r.getLearningRate(), 1e-6); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l0.getDropOut()); assertEquals(0.1, TestUtils.getL1(l0), 1e-6); assertEquals(new WeightDecay(0.2,false), TestUtils.getWeightDecayReg(l0)); @@ -131,11 +131,11 @@ public class RegressionTest080 extends BaseDL4JTest { assertEquals(4, l1.getNIn()); assertEquals(5, l1.getNOut()); assertEquals(new WeightInitDistribution(new NormalDistribution(0.1, 1.2)), l1.getWeightInit()); - assertTrue(l1.getIUpdater() instanceof RmsProp); - r = (RmsProp) l1.getIUpdater(); + assertTrue(l1.getUpdater() instanceof RmsProp); + r = (RmsProp) l1.getUpdater(); assertEquals(0.96, r.getRmsDecay(), 1e-6); assertEquals(0.15, r.getLearningRate(), 1e-6); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertEquals(new Dropout(0.6), l1.getDropOut()); assertEquals(0.1, TestUtils.getL1(l1), 1e-6); assertEquals(new WeightDecay(0.2, false), TestUtils.getWeightDecayReg(l1)); @@ -163,11 +163,11 @@ public class RegressionTest080 extends BaseDL4JTest { assertEquals(3, l0.getNIn()); assertEquals(3, l0.getNOut()); assertEquals(new WeightInitRelu(), l0.getWeightInit()); - assertTrue(l0.getIUpdater() instanceof RmsProp); - RmsProp r = (RmsProp) l0.getIUpdater(); + assertTrue(l0.getUpdater() instanceof RmsProp); + RmsProp r = (RmsProp) l0.getUpdater(); assertEquals(0.96, r.getRmsDecay(), 1e-6); assertEquals(0.15, r.getLearningRate(), 1e-6); - assertEquals(0.15, ((RmsProp)l0.getIUpdater()).getLearningRate(), 1e-6); + assertEquals(0.15, ((RmsProp)l0.getUpdater()).getLearningRate(), 1e-6); assertArrayEquals(new int[] {2, 2}, l0.getKernelSize()); assertArrayEquals(new int[] {1, 1}, l0.getStride()); assertArrayEquals(new int[] {0, 0}, l0.getPadding()); @@ -186,8 +186,8 @@ public class RegressionTest080 extends BaseDL4JTest { assertEquals(26 * 26 * 3, l2.getNIn()); assertEquals(5, l2.getNOut()); assertEquals(new WeightInitRelu(), l2.getWeightInit()); - assertTrue(l2.getIUpdater() instanceof RmsProp); - r = (RmsProp) l2.getIUpdater(); + assertTrue(l2.getUpdater() instanceof RmsProp); + r = (RmsProp) l2.getUpdater(); assertEquals(0.96, r.getRmsDecay(), 1e-6); assertEquals(0.15, r.getLearningRate(), 1e-6); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java index 6555c5eec..3ea3b540d 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java @@ -91,21 +91,21 @@ public class RegressionTest100a extends BaseDL4JTest { assertEquals(200, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new RmsProp(0.1), l0.getIUpdater()); + assertEquals(new RmsProp(0.1), l0.getUpdater()); GravesLSTM l1 = (GravesLSTM) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(200, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l1)); - assertEquals(new RmsProp(0.1), l1.getIUpdater()); + assertEquals(new RmsProp(0.1), l1.getUpdater()); RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).getLayerConfiguration(); assertEquals(new ActivationSoftmax(), l2.getActivationFn()); assertEquals(77, l2.getNOut()); assertEquals(new WeightInitXavier(), l2.getWeightInit()); assertEquals(new WeightDecay(0.001, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new RmsProp(0.1), l0.getIUpdater()); + assertEquals(new RmsProp(0.1), l0.getUpdater()); assertEquals(BackpropType.TruncatedBPTT, net.getNetConfiguration().getBackpropType()); assertEquals(50, net.getNetConfiguration().getTbpttBackLength()); @@ -141,7 +141,7 @@ public class RegressionTest100a extends BaseDL4JTest { assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new WeightDecay(1e-4, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new Adam(0.05), l0.getIUpdater()); + assertEquals(new Adam(0.05), l0.getUpdater()); INDArray outExp; File f2 = Resources.asFile("regression_testing/100a/VaeMNISTAnomaly_Output_100a.bin"); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java index 223b7be91..ce94238cd 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java @@ -75,12 +75,12 @@ public class RegressionTest100b3 extends BaseDL4JTest { DenseLayer l0 = (DenseLayer) net.getLayer(0).getLayerConfiguration(); assertEquals(new ActivationTanH(), l0.getActivationFn()); assertEquals(new WeightDecay(0.03, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new RmsProp(0.95), l0.getIUpdater()); + assertEquals(new RmsProp(0.95), l0.getUpdater()); CustomLayer l1 = (CustomLayer) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(new ActivationSigmoid(), l1.getSecondActivationFunction()); - assertEquals(new RmsProp(0.95), l1.getIUpdater()); + assertEquals(new RmsProp(0.95), l1.getUpdater()); INDArray outExp; @@ -126,21 +126,21 @@ public class RegressionTest100b3 extends BaseDL4JTest { assertEquals(200, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new WeightDecay(0.0001, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new Adam(0.005), l0.getIUpdater()); + assertEquals(new Adam(0.005), l0.getUpdater()); LSTM l1 = (LSTM) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(200, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); assertEquals(new WeightDecay(0.0001, false), TestUtils.getWeightDecayReg(l1)); - assertEquals(new Adam(0.005), l1.getIUpdater()); + assertEquals(new Adam(0.005), l1.getUpdater()); RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).getLayerConfiguration(); assertEquals(new ActivationSoftmax(), l2.getActivationFn()); assertEquals(77, l2.getNOut()); assertEquals(new WeightInitXavier(), l2.getWeightInit()); assertEquals(new WeightDecay(0.0001, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new Adam(0.005), l0.getIUpdater()); + assertEquals(new Adam(0.005), l0.getUpdater()); assertEquals(BackpropType.TruncatedBPTT, net.getNetConfiguration().getBackpropType()); assertEquals(50, net.getNetConfiguration().getTbpttBackLength()); @@ -176,7 +176,7 @@ public class RegressionTest100b3 extends BaseDL4JTest { assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new WeightDecay(1e-4, false), TestUtils.getWeightDecayReg(l0)); - assertEquals(new Adam(1e-3), l0.getIUpdater()); + assertEquals(new Adam(1e-3), l0.getUpdater()); INDArray outExp; File f2 = Resources.asFile("regression_testing/100b3/VaeMNISTAnomaly_Output_100b3.bin"); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java index 65329cb4a..6a898dae5 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java @@ -94,12 +94,12 @@ public class RegressionTest100b4 extends BaseDL4JTest { DenseLayer l0 = (DenseLayer) net.getLayer(0).getLayerConfiguration(); assertEquals(new ActivationTanH(), l0.getActivationFn()); assertEquals(new L2Regularization(0.03), TestUtils.getL2Reg(l0)); - assertEquals(new RmsProp(0.95), l0.getIUpdater()); + assertEquals(new RmsProp(0.95), l0.getUpdater()); CustomLayer l1 = (CustomLayer) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(new ActivationSigmoid(), l1.getSecondActivationFunction()); - assertEquals(new RmsProp(0.95), l1.getIUpdater()); + assertEquals(new RmsProp(0.95), l1.getUpdater()); INDArray outExp; File f2 = Resources @@ -144,21 +144,21 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertEquals(200, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0)); - assertEquals(new Adam(0.005), l0.getIUpdater()); + assertEquals(new Adam(0.005), l0.getUpdater()); LSTM l1 = (LSTM) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(200, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l1)); - assertEquals(new Adam(0.005), l1.getIUpdater()); + assertEquals(new Adam(0.005), l1.getUpdater()); RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).getLayerConfiguration(); assertEquals(new ActivationSoftmax(), l2.getActivationFn()); assertEquals(77, l2.getNOut()); assertEquals(new WeightInitXavier(), l2.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l2)); - assertEquals(new Adam(0.005), l2.getIUpdater()); + assertEquals(new Adam(0.005), l2.getUpdater()); assertEquals(BackpropType.TruncatedBPTT, net.getNetConfiguration().getBackpropType()); assertEquals(50, net.getNetConfiguration().getTbpttBackLength()); @@ -194,7 +194,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0)); - assertEquals(new Adam(1e-3), l0.getIUpdater()); + assertEquals(new Adam(1e-3), l0.getUpdater()); INDArray outExp; File f2 = Resources.asFile("regression_testing/100b4/VaeMNISTAnomaly_Output_100b4.bin"); @@ -262,7 +262,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertEquals(4, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0)); - assertEquals(new Adam(0.005), l0.getIUpdater()); + assertEquals(new Adam(0.005), l0.getUpdater()); assertArrayEquals(new int[]{3, 3}, l0.getKernelSize()); assertArrayEquals(new int[]{2, 1}, l0.getStride()); assertArrayEquals(new int[]{1, 1}, l0.getDilation()); @@ -273,7 +273,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertEquals(8, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l1)); - assertEquals(new Adam(0.005), l1.getIUpdater()); + assertEquals(new Adam(0.005), l1.getUpdater()); assertArrayEquals(new int[]{3, 3}, l1.getKernelSize()); assertArrayEquals(new int[]{1, 1}, l1.getStride()); assertArrayEquals(new int[]{1, 1}, l1.getDilation()); @@ -299,7 +299,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertEquals(16, l5.getNOut()); assertEquals(new WeightInitXavier(), l5.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l5)); - assertEquals(new Adam(0.005), l5.getIUpdater()); + assertEquals(new Adam(0.005), l5.getUpdater()); assertArrayEquals(new int[]{3, 3}, l5.getKernelSize()); assertArrayEquals(new int[]{1, 1}, l5.getStride()); assertArrayEquals(new int[]{1, 1}, l5.getDilation()); @@ -320,7 +320,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertEquals(4, l8.getNOut()); assertEquals(new WeightInitXavier(), l8.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l8)); - assertEquals(new Adam(0.005), l8.getIUpdater()); + assertEquals(new Adam(0.005), l8.getUpdater()); assertArrayEquals(new int[]{4, 4}, l8.getKernelSize()); assertArrayEquals(new int[]{1, 1}, l8.getStride()); assertArrayEquals(new int[]{1, 1}, l8.getDilation()); @@ -329,7 +329,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { CnnLossLayer l9 = (CnnLossLayer) net.getLayer(9).getLayerConfiguration(); assertEquals(new WeightInitXavier(), l9.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l9)); - assertEquals(new Adam(0.005), l9.getIUpdater()); + assertEquals(new Adam(0.005), l9.getUpdater()); assertEquals(new LossMAE(), l9.getLossFunction()); INDArray outExp; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java index 0a43d9769..e8ab99c2b 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java @@ -76,12 +76,12 @@ public class RegressionTest100b6 extends BaseDL4JTest { DenseLayer l0 = (DenseLayer) net.getLayer(0).getLayerConfiguration(); assertEquals(new ActivationTanH(), l0.getActivationFn()); assertEquals(new L2Regularization(0.03), TestUtils.getL2Reg(l0)); - assertEquals(new RmsProp(0.95), l0.getIUpdater()); + assertEquals(new RmsProp(0.95), l0.getUpdater()); CustomLayer l1 = (CustomLayer) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(new ActivationSigmoid(), l1.getSecondActivationFunction()); - assertEquals(new RmsProp(0.95), l1.getIUpdater()); + assertEquals(new RmsProp(0.95), l1.getUpdater()); INDArray outExp; File f2 = Resources @@ -126,21 +126,21 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertEquals(200, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0)); - assertEquals(new Adam(0.005), l0.getIUpdater()); + assertEquals(new Adam(0.005), l0.getUpdater()); LSTM l1 = (LSTM) net.getLayer(1).getLayerConfiguration(); assertEquals(new ActivationTanH(), l1.getActivationFn()); assertEquals(200, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l1)); - assertEquals(new Adam(0.005), l1.getIUpdater()); + assertEquals(new Adam(0.005), l1.getUpdater()); RnnOutputLayer l2 = (RnnOutputLayer) net.getLayer(2).getLayerConfiguration(); assertEquals(new ActivationSoftmax(), l2.getActivationFn()); assertEquals(77, l2.getNOut()); assertEquals(new WeightInitXavier(), l2.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l2)); - assertEquals(new Adam(0.005), l2.getIUpdater()); + assertEquals(new Adam(0.005), l2.getUpdater()); assertEquals(BackpropType.TruncatedBPTT, net.getNetConfiguration().getBackpropType()); assertEquals(50, net.getNetConfiguration().getTbpttBackLength()); @@ -176,7 +176,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertArrayEquals(new int[]{256, 256}, l0.getDecoderLayerSizes()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0)); - assertEquals(new Adam(1e-3), l0.getIUpdater()); + assertEquals(new Adam(1e-3), l0.getUpdater()); INDArray outExp; File f2 = Resources.asFile("regression_testing/100b6/VaeMNISTAnomaly_Output_100b6.bin"); @@ -242,7 +242,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertEquals(4, l0.getNOut()); assertEquals(new WeightInitXavier(), l0.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l0)); - assertEquals(new Adam(0.005), l0.getIUpdater()); + assertEquals(new Adam(0.005), l0.getUpdater()); assertArrayEquals(new int[]{3, 3}, l0.getKernelSize()); assertArrayEquals(new int[]{2, 1}, l0.getStride()); assertArrayEquals(new int[]{1, 1}, l0.getDilation()); @@ -253,7 +253,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertEquals(8, l1.getNOut()); assertEquals(new WeightInitXavier(), l1.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l1)); - assertEquals(new Adam(0.005), l1.getIUpdater()); + assertEquals(new Adam(0.005), l1.getUpdater()); assertArrayEquals(new int[]{3, 3}, l1.getKernelSize()); assertArrayEquals(new int[]{1, 1}, l1.getStride()); assertArrayEquals(new int[]{1, 1}, l1.getDilation()); @@ -279,7 +279,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertEquals(16, l5.getNOut()); assertEquals(new WeightInitXavier(), l5.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l5)); - assertEquals(new Adam(0.005), l5.getIUpdater()); + assertEquals(new Adam(0.005), l5.getUpdater()); assertArrayEquals(new int[]{3, 3}, l5.getKernelSize()); assertArrayEquals(new int[]{1, 1}, l5.getStride()); assertArrayEquals(new int[]{1, 1}, l5.getDilation()); @@ -300,7 +300,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertEquals(4, l8.getNOut()); assertEquals(new WeightInitXavier(), l8.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l8)); - assertEquals(new Adam(0.005), l8.getIUpdater()); + assertEquals(new Adam(0.005), l8.getUpdater()); assertArrayEquals(new int[]{4, 4}, l8.getKernelSize()); assertArrayEquals(new int[]{1, 1}, l8.getStride()); assertArrayEquals(new int[]{1, 1}, l8.getDilation()); @@ -309,7 +309,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { CnnLossLayer l9 = (CnnLossLayer) net.getLayer(9).getLayerConfiguration(); assertEquals(new WeightInitXavier(), l9.getWeightInit()); assertEquals(new L2Regularization(0.0001), TestUtils.getL2Reg(l9)); - assertEquals(new Adam(0.005), l9.getIUpdater()); + assertEquals(new Adam(0.005), l9.getUpdater()); assertEquals(new LossMAE(), l9.getLossFunction()); INDArray outExp; diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java index 6eb8e1ae2..d34173e56 100644 --- a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java @@ -113,7 +113,7 @@ public class CustomLayer extends FeedForwardLayer { InputType outputType = getOutputType(-1, inputType); val numParams = initializer().numParams(this); - int updaterStateSize = (int) getIUpdater().stateSize(numParams); + int updaterStateSize = (int) getUpdater().stateSize(numParams); int trainSizeFixed = 0; int trainSizeVariable = 0; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java index a2db2946b..a85d285fe 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java @@ -235,7 +235,7 @@ public class GradientCheckUtil { for (LayerConfiguration n : c.net.getNetConfiguration().getFlattenedLayerConfigurations()) { if (n instanceof BaseLayerConfiguration) { BaseLayerConfiguration bl = (BaseLayerConfiguration) n; - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); if (u instanceof Sgd) { // Must have LR of 1.0 double lr = ((Sgd) u).getLearningRate(); @@ -540,7 +540,7 @@ public class GradientCheckUtil { if (lv.getLayerConfiguration() instanceof BaseLayerConfiguration) { BaseLayerConfiguration bl = (BaseLayerConfiguration) lv.getLayerConfiguration(); - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); if (u instanceof Sgd) { // Must have LR of 1.0 double lr = ((Sgd) u).getLearningRate(); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetBaseBuilderConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetBaseBuilderConfiguration.java index a9afe9332..578f94fd7 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetBaseBuilderConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetBaseBuilderConfiguration.java @@ -322,7 +322,7 @@ public abstract class NeuralNetBaseBuilderConfiguration implements INeuralNetwor @Getter @Setter @lombok.Builder.Default @JsonIgnore private IUpdater iUpdater = new Sgd(); /** * Gradient updater configuration, for the biases only. If not set, biases will use the updater as - * set by {@link #setIUpdater(IUpdater)}
+ * set by {@link #setUpdater(IUpdater)}
* Note: values set by this method will be applied to all applicable layers in the network, unless * a different value is explicitly set on a given layer. In other words: values set via this * method are used as the default value, and can be overridden on a per-layer basis. diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java index 92d181533..17c1631a8 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java @@ -65,7 +65,7 @@ public class ActivationLayer extends NoParamLayer { } @Override - public IUpdater getIUpdater() { + public IUpdater getUpdater() { return null; } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java index 54cb41d71..0ed7a14fa 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java @@ -88,7 +88,7 @@ public class AutoEncoder extends BasePretrainNetwork { val actElementsPerEx = outputType.arrayElementsPerExample() + inputType.arrayElementsPerExample(); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); int trainSizePerEx = 0; if (getDropOut() != null) { diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayerConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayerConfiguration.java index bd415e22b..9bdc63e4f 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayerConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayerConfiguration.java @@ -95,7 +95,7 @@ public abstract class BaseLayerConfiguration extends LayerConfiguration * Gradient updater. For example, {@link org.nd4j.linalg.learning.config.Adam} or {@link * org.nd4j.linalg.learning.config.Nesterovs} */ - @Getter @Setter + @Getter protected IUpdater updater; /** * Gradient updater configuration, for the biases only. If not set, biases will use the updater as @@ -134,7 +134,7 @@ public abstract class BaseLayerConfiguration extends LayerConfiguration */ public void resetLayerDefaultConfig() { // clear the learning related params for all layers in the origConf and set to defaults - this.setIUpdater(null); + this.setUpdater( (IUpdater) null); this.setWeightInit(null); this.setBiasInit(Double.NaN); this.setGainInit(Double.NaN); @@ -142,10 +142,16 @@ public abstract class BaseLayerConfiguration extends LayerConfiguration this.regularizationBias = null; this.setGradientNormalization(GradientNormalization.None); this.setGradientNormalizationThreshold(1.0); - this.updater = null; this.biasUpdater = null; } + public void setUpdater(Updater updater) { + setUpdater(updater.getIUpdaterWithDefaultConfig()); + } + public void setUpdater(IUpdater iUpdater) { + this.updater=iUpdater; + } + @Override public BaseLayerConfiguration clone() { BaseLayerConfiguration clone = (BaseLayerConfiguration) super.clone(); @@ -203,6 +209,7 @@ public abstract class BaseLayerConfiguration extends LayerConfiguration if (this.updater == null) this.updater = conf.getUpdater(); if (this.regularizationBias == null) this.regularizationBias = conf.getRegularizationBias(); if (this.regularization == null) this.regularization = conf.getRegularization(); + if( this.weightInit == null) this.weightInit = conf.getWeightInit(); if (this.gradientNormalization == null) this.gradientNormalization = conf.getGradientNormalization(); // if(this.weightInit == null) this.weightInit = conf.getWeightInit(); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java index a24720130..f8441388c 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java @@ -56,7 +56,7 @@ public abstract class BaseOutputLayer extends FeedForwardLayer { InputType outputType = getOutputType(-1, inputType); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); int trainSizeFixed = 0; int trainSizeVariable = 0; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java index d1afb28a3..351aea24c 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java @@ -235,7 +235,7 @@ public class ConvolutionLayer extends FeedForwardLayer { @Override public LayerMemoryReport getMemoryReport(InputType inputType) { val paramSize = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(paramSize); + val updaterStateSize = (int) getUpdater().stateSize(paramSize); InputType.InputTypeConvolutional c = (InputType.InputTypeConvolutional) inputType; InputType.InputTypeConvolutional outputType = diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java index 5a661065a..ce34310ed 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java @@ -60,7 +60,7 @@ public class DenseLayer extends FeedForwardLayer { LayerValidation.assertNInNOutSet( "DenseLayerConfiguration", getName(), layerIndex, getNIn(), getNOut()); LayerConfiguration lconf = conf.getFlattenedLayerConfigurations().get(layerIndex); - lconf.runInheritance(); + runInheritance(); org.deeplearning4j.nn.layers.feedforward.dense.DenseLayer ret = new org.deeplearning4j.nn.layers.feedforward.dense.DenseLayer(lconf, networkDataType); @@ -84,7 +84,7 @@ public class DenseLayer extends FeedForwardLayer { InputType outputType = getOutputType(-1, inputType); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); int trainSizeFixed = 0; int trainSizeVariable = 0; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java index 2f535bce3..bb5286e92 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java @@ -96,7 +96,7 @@ public class EmbeddingLayer extends FeedForwardLayer { val actElementsPerEx = outputType.arrayElementsPerExample(); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); // Embedding layer does not use caching. // Inference: no working memory - just activations (pullRows) diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java index e21593fb6..a720a81a9 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java @@ -162,7 +162,7 @@ extends FeedForwardLayerBuilder { val actElementsPerEx = outputType.arrayElementsPerExample(); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); return new LayerMemoryReport.Builder(name, EmbeddingSequenceLayer.class, inputType, outputType) .standardMemory(numParams, updaterStateSize).workingMemory(0, 0, 0, actElementsPerEx) diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerConfiguration.java index a34d5dc51..eb24b60ce 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerConfiguration.java @@ -34,6 +34,7 @@ import org.deeplearning4j.nn.api.ParamInitializer; import org.deeplearning4j.nn.api.layers.LayerConstraint; import org.deeplearning4j.nn.conf.InputPreProcessor; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.Updater; import org.deeplearning4j.nn.conf.dropout.Dropout; import org.deeplearning4j.nn.conf.dropout.IDropout; import org.deeplearning4j.nn.conf.inputs.InputType; @@ -281,16 +282,6 @@ public abstract class LayerConfiguration "Not supported: all layers with parameters should override this method"); } - public IUpdater getIUpdater() { - throw new UnsupportedOperationException( - "Not supported: all layers with parameters should override this method"); - } - - public void setIUpdater(IUpdater iUpdater) { - log.warn( - "Setting an IUpdater on {} with name {} has no effect.", getClass().getSimpleName(), name); - } - /** * This is a report of the estimated memory consumption for the given layer * @@ -316,6 +307,7 @@ public abstract class LayerConfiguration if (this.activation == null) this.activation = conf.getActivation(); if (this.dropOut == null) this.dropOut = conf.getIdropOut(); if (this.weightNoise == null) this.weightNoise = conf.getWeightNoise(); + } /** @@ -326,6 +318,24 @@ public abstract class LayerConfiguration runInheritance(getNetConfiguration()); } + /** + * This will always return no-Op updater. + * @return + */ + public IUpdater getUpdater() { + log.warn("Calling getUpdater() in {} will always return no-Op Updater.", LayerConfiguration.class.getSimpleName()); + return Updater.NONE.getIUpdaterWithDefaultConfig(); + } + @Deprecated + public void setUpdater(Updater updater) { + setUpdater(updater.getIUpdaterWithDefaultConfig()); + } + + public void setUpdater(IUpdater iUpdater) { + throw new RuntimeException("When " + this.getName() + " wants to you an Updater, it needs to override the " + + "Getter/ Setter for the Updater and not rely on LayerConfiguration class."); + } + public abstract static class LayerConfigurationBuilder< C extends LayerConfiguration, B extends LayerConfigurationBuilder> { diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java index 140807623..9d3352dac 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java @@ -249,17 +249,6 @@ public class LocallyConnected1D extends SameDiffLayer { } } - @Override - public void applyGlobalConfigToLayer( - NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) { - NeuralNetConfiguration global_conf = globalConfig.build(); - if (activation == null) { - activation = SameDiffLayerUtils.fromIActivation(global_conf.getActivation()); - } - if (convolutionMode == null) { - convolutionMode = global_conf.getConvolutionMode(); - } - } private static final class LocallyConnected1DBuilderImpl extends LocallyConnected1DBuilder { diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java index 04922b857..83dde1e49 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java @@ -305,17 +305,6 @@ public class LocallyConnected2D extends SameDiffLayer { } } - @Override - public void applyGlobalConfigToLayer( - NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) { - NeuralNetConfiguration gconf = globalConfig.build(); - if (activation == null) { - activation = SameDiffLayerUtils.fromIActivation(gconf.getActivation()); - } - if (convolutionMode == null) { - convolutionMode = gconf.getConvolutionMode(); - } - } private static final class LocallyConnected2DBuilderImpl extends LocallyConnected2DBuilder { diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java index 78ae44d93..ab77c3012 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java @@ -56,10 +56,11 @@ public abstract class NoParamLayer extends LayerConfiguration { } /** + * Will always return no-Op updater. * @return */ @Override - public IUpdater getIUpdater() { + public IUpdater getUpdater() { return Updater.NONE.getIUpdaterWithDefaultConfig(); } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java index 3d1da835a..40ec2808d 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java @@ -40,9 +40,7 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; @SuperBuilder(builderMethodName = "innerBuilder") public class OutputLayer extends BaseOutputLayer { - { // Set default activation function to softmax (to match default loss function MCXENT) - setActivation(Activation.SOFTMAX.getActivationFunction()); - } + public static OutputLayerBuilder builder() { return innerBuilder(); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java index cacd65ca5..4244c252c 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java @@ -120,7 +120,7 @@ public class PReLULayer extends BaseLayerConfiguration { InputType outputType = getOutputType(-1, inputType); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); return new LayerMemoryReport.Builder(name, PReLULayer.class, inputType, outputType) .standardMemory(numParams, updaterStateSize) diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java index a57ab343c..dedebd23c 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java @@ -22,6 +22,7 @@ package org.deeplearning4j.nn.conf.layers; import lombok.*; import lombok.experimental.SuperBuilder; +import org.deeplearning4j.nn.api.ITraininableLayerConfiguration; import org.deeplearning4j.nn.conf.InputPreProcessor; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.RNNFormat; @@ -35,6 +36,7 @@ import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -44,7 +46,9 @@ import java.util.Map; @Data @EqualsAndHashCode(callSuper = true) @SuperBuilder -public class RecurrentAttentionLayer extends SameDiffLayer { +public class RecurrentAttentionLayer extends SameDiffLayer implements ITraininableLayerConfiguration { + +private DataType dataType; private static final class RecurrentAttentionLayerBuilderImpl extends RecurrentAttentionLayerBuilder { public RecurrentAttentionLayer build() { @@ -190,13 +194,6 @@ public class RecurrentAttentionLayer extends SameDiffLayer { } } - @Override - public void applyGlobalConfigToLayer(NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) { - if (activation == null) { - activation = SameDiffLayerUtils.fromIActivation(globalConfig.build().getActivation()); - } - } - @Override public void validateInput(INDArray input) { final long inputLength = input.size(2); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java index 65c47b7f5..6865119e4 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java @@ -91,7 +91,7 @@ public class ElementWiseMultiplicationLayer extends org.deeplearning4j.nn.conf.l InputType outputType = getOutputType(-1, inputType); val numParams = initializer().numParams(this); - val updaterStateSize = (int) getIUpdater().stateSize(numParams); + val updaterStateSize = (int) getUpdater().stateSize(numParams); int trainSizeFixed = 0; int trainSizeVariable = 0; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java index ff9a66c2b..b7f42a5cf 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java @@ -82,6 +82,7 @@ public class FrozenLayerWithBackprop extends BaseWrapperLayerConfiguration { org.deeplearning4j.nn.api.Layer newUnderlyingLayer = underlying.instantiate(conf, trainingListeners, layerIndex, layerParamsView, initializeParams, networkDataType); + runInheritance(); newUnderlyingLayer.setLayerConfiguration(underlying); //Fix a problem, where the embedded layer gets the conf of the frozen layer, rather than its own NeuralNetConfiguration nncUnderlying = underlying.getNetConfiguration(); @@ -130,4 +131,6 @@ public class FrozenLayerWithBackprop extends BaseWrapperLayerConfiguration { this.constraints = constraints; this.underlying.setConstraints(constraints); } + + } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java index 70cf38d07..12239673e 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java @@ -38,6 +38,7 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.nn.weights.WeightInitUtil; import org.deeplearning4j.optimize.api.TrainingListener; import org.deeplearning4j.util.NetworkUtils; +import org.jetbrains.annotations.NotNull; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -93,6 +94,21 @@ public abstract class AbstractSameDiffLayer extends LayerConfiguration { @Getter @Setter private SDLayerParams layerParams; + @Override + public void runInheritance(@NotNull NeuralNetConfiguration conf) { + super.runInheritance(conf); + if (this.biasUpdater == null ) this.biasUpdater = conf.getBiasUpdater(); + if (this.updater == null) this.updater = conf.getUpdater(); + if (this.regularizationBias == null || regularizationBias.isEmpty()) this.regularizationBias = conf.getRegularizationBias(); + if (this.regularization == null || regularization.isEmpty()) this.regularization = conf.getRegularization(); + // if( this.weightInit == null) this.weightInit = conf.getWeightInit(); + if (this.gradientNormalization == null) + this.gradientNormalization = conf.getGradientNormalization(); + // if(this.weightInit == null) this.weightInit = conf.getWeightInit(); + if (Double.isNaN(gradientNormalizationThreshold)) { + this.gradientNormalizationThreshold = conf.getGradientNormalizationThreshold(); + } + } @Override public List getRegularizationByParam(String paramName) { if (layerParams.isWeightParam(paramName)) { @@ -122,10 +138,6 @@ public abstract class AbstractSameDiffLayer extends LayerConfiguration { return null; } - public void applyGlobalConfigToLayer( - NeuralNetConfiguration.NeuralNetConfigurationBuilder globalConfig) { - // Default implementation: no op - } /** * Define the parameters for the network. Use {@link SDLayerParams#addWeightParam(String, @@ -195,29 +207,6 @@ public abstract class AbstractSameDiffLayer extends LayerConfiguration { fanIn, fanOut, array.shape(), weightInit, null, paramReshapeOrder(null), array); } - public void applyGlobalConfig(NeuralNetConfiguration.NeuralNetConfigurationBuilder b) { - NeuralNetConfiguration bConf = b.build(); - if (regularization == null || regularization.isEmpty()) { - regularization = bConf.getRegularization(); - } - if (regularizationBias == null || regularizationBias.isEmpty()) { - regularizationBias = bConf.getRegularizationBias(); - } - if (updater == null) { - updater = bConf.getUpdater(); - } - if (biasUpdater == null) { - biasUpdater = bConf.getBiasUpdater(); - } - if (gradientNormalization == null) { - gradientNormalization = bConf.getGradientNormalization(); - } - if (Double.isNaN(gradientNormalizationThreshold)) { - gradientNormalizationThreshold = bConf.getGradientNormalizationThreshold(); - } - - applyGlobalConfigToLayer(b); - } /** * This method generates an "all ones" mask array for use in the SameDiff model when none is diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java index 53dd8dffb..bf2ae8255 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java @@ -137,7 +137,7 @@ public class VariationalAutoencoder extends BasePretrainNetwork { val actElementsPerEx = outputType.arrayElementsPerExample(); val numParams = initializer().numParams(this); - int updaterStateSize = (int) getIUpdater().stateSize(numParams); + int updaterStateSize = (int) getUpdater().stateSize(numParams); int inferenceWorkingMemSizePerEx = 0; // Forward pass size through the encoder: diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayerConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayerConfiguration.java index 72a80d2d8..72f4fc659 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayerConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayerConfiguration.java @@ -123,17 +123,17 @@ public abstract class BaseWrapperLayerConfiguration extends LayerConfiguration { /** * @return */ - @Override - public IUpdater getIUpdater() { - return underlying.getIUpdater(); + + public IUpdater getUpdater() { + return underlying.getUpdater(); } /** * @param iUpdater */ - @Override - public void setIUpdater(IUpdater iUpdater) { - underlying.setIUpdater(iUpdater); + + public void setUpdater(IUpdater iUpdater) { + underlying.setUpdater(iUpdater); } @Override diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java index 3099ffc00..01e1fc00a 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java @@ -68,7 +68,7 @@ public abstract class BaseNetConfigDeserializer extends StdDeserializer im for(LayerConfiguration l : layers){ if(l instanceof BaseLayerConfiguration){ BaseLayerConfiguration bl = (BaseLayerConfiguration)l; - if(bl.getIUpdater() == null && bl.initializer().numParams(bl) > 0){ + if(bl.getUpdater() == null && bl.initializer().numParams(bl) > 0){ return true; } } @@ -200,7 +200,7 @@ public abstract class BaseNetConfigDeserializer extends StdDeserializer im break; } - layer.setIUpdater(iu); + layer.setUpdater(iu); } } } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java index e3a6ba1cd..a3ea58079 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java @@ -116,7 +116,7 @@ public class ComputationGraphConfigurationDeserializer } if(attemptIUpdaterFromLegacy && layers[layerIdx] instanceof BaseLayerConfiguration - && ((BaseLayerConfiguration)layers[layerIdx]).getIUpdater() == null){ + && ((BaseLayerConfiguration)layers[layerIdx]).getUpdater() == null){ handleUpdaterBackwardCompatibility((BaseLayerConfiguration)layers[layerIdx], (ObjectNode)next); } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/NeuralNetConfigurationDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/NeuralNetConfigurationDeserializer.java index e396df456..08d7f4c6c 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/NeuralNetConfigurationDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/NeuralNetConfigurationDeserializer.java @@ -87,7 +87,7 @@ public class NeuralNetConfigurationDeserializer extends BaseNetConfigDeserialize ObjectNode on = (ObjectNode) confsNode.get(i); ObjectNode confNode = null; if(layers[i] instanceof BaseLayerConfiguration - && ((BaseLayerConfiguration)layers[i]).getIUpdater() == null){ + && ((BaseLayerConfiguration)layers[i]).getUpdater() == null){ //layer -> (first/only child) -> updater if(on.has("layer")){ confNode = on; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java index fdf01ad66..cccb3d476 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java @@ -77,6 +77,7 @@ public class WeightNoise implements IWeightNoise { (applyToBias && init.isBiasParam(layer.getLayerConfiguration(), paramKey))) { org.nd4j.linalg.api.rng.distribution.Distribution dist = Distributions.createDistribution(distribution); + INDArray noise = dist.sample(param.ulike()); INDArray out = workspaceMgr.createUninitialized(ArrayType.INPUT, param.dataType(), param.shape(), param.ordering()); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java index 5d8141549..3096b27c3 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java @@ -577,7 +577,8 @@ public abstract class AbstractLayer impl */ @Override public Map getParamTable(boolean isBackprop) { - throw new RuntimeException("Not implemented"); +// throw new RuntimeException("Not implemented"); + return null; } /** diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java index f4269e52e..94e3b6de8 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java @@ -340,6 +340,6 @@ public class GlobalPoolingLayer extends AbstractLayer paramTable) { - throw new RuntimeException("Not implemented."); + } } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java index 2ee627263..c7c7fa643 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java @@ -768,7 +768,7 @@ public class LSTMHelpers { InputType outputType = lstmLayer.getOutputType(-1, inputType); val numParams = lstmLayer.initializer().numParams(lstmLayer); - int updaterSize = (int) lstmLayer.getIUpdater().stateSize(numParams); + int updaterSize = (int) lstmLayer.getUpdater().stateSize(numParams); //Memory use during forward pass: //ifogActivations: nTimeSteps * [minibatch,4*layerSize] (not cached during inference fwd pass) diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java index bca8e7e68..650bad51f 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java @@ -565,7 +565,7 @@ public class MultiLayerNetwork extends ArtificialNeuralNetwork * inference/backprop are excluded from the returned list. * * @param backpropParamsOnly If true, return backprop params only. If false: return all params - * @return Parameters for the network + * @return Parameters for the network, empty Map if no parameters present in the neural network */ @Override public Map getParamTable(boolean backpropParamsOnly) { @@ -573,10 +573,11 @@ public class MultiLayerNetwork extends ArtificialNeuralNetwork Map allParams = new LinkedHashMap<>(); for (int i = 0; i < layers.length; i++) { Map paramMap = layers[i].getParamTable(backpropParamsOnly); + if(paramMap!=null){ for (Map.Entry entry : paramMap.entrySet()) { String newKey = i + "_" + entry.getKey(); allParams.put(newKey, entry.getValue()); - } + }} } return allParams; } diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java index 2278af04a..77c2b3a5f 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java @@ -94,6 +94,7 @@ public class DefaultParamInitializer extends AbstractParamInitializer { if (!(conf instanceof org.deeplearning4j.nn.conf.layers.FeedForwardLayer)) throw new IllegalArgumentException("unsupported layer type: " + conf.getClass().getName()); + INDArray reshapedParamsView = paramsView.reshape(paramsView.length()); Map params = Collections.synchronizedMap(new LinkedHashMap()); val length = numParams(conf); @@ -107,14 +108,14 @@ public class DefaultParamInitializer extends AbstractParamInitializer { val nOut = layerConf.getNOut(); val nWeightParams = nIn * nOut; - INDArray weightView = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nWeightParams)); + INDArray weightView = reshapedParamsView.get(NDArrayIndex.interval(0, nWeightParams)); params.put(WEIGHT_KEY, createWeightMatrix(layerConf, weightView, initializeParams)); layerConf.addVariable(WEIGHT_KEY); long offset = nWeightParams; if(hasBias(layerConf)){ - INDArray biasView = paramsView.get(NDArrayIndex.interval(0,0,true), + INDArray biasView = reshapedParamsView.get( NDArrayIndex.interval(offset, offset + nOut)); params.put(BIAS_KEY, createBias(layerConf, biasView, initializeParams)); layerConf.addVariable(BIAS_KEY); @@ -122,7 +123,7 @@ public class DefaultParamInitializer extends AbstractParamInitializer { } if(hasLayerNorm(layerConf)){ - INDArray gainView = paramsView.get(NDArrayIndex.interval(0,0,true), + INDArray gainView = reshapedParamsView.get( NDArrayIndex.interval(offset, offset + nOut)); params.put(GAIN_KEY, createGain(conf, gainView, initializeParams)); conf.addVariable(GAIN_KEY); @@ -138,23 +139,24 @@ public class DefaultParamInitializer extends AbstractParamInitializer { val nIn = layerConf.getNIn(); val nOut = layerConf.getNOut(); val nWeightParams = nIn * nOut; + INDArray gradientViewReshaped = gradientView.reshape(gradientView.length()); - INDArray weightGradientView = gradientView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nWeightParams)) - .reshape('f', nIn, nOut); + INDArray weightGradientView = gradientViewReshaped.get(NDArrayIndex.interval(0, nWeightParams)) + .reshape('f', nIn, nOut); Map out = new LinkedHashMap<>(); out.put(WEIGHT_KEY, weightGradientView); long offset = nWeightParams; if(hasBias(layerConf)){ - INDArray biasView = gradientView.get(NDArrayIndex.interval(0,0,true), + INDArray biasView = gradientViewReshaped.get( NDArrayIndex.interval(offset, offset + nOut)); //Already a row vector out.put(BIAS_KEY, biasView); offset += nOut; } if(hasLayerNorm(layerConf)){ - INDArray gainView = gradientView.get(NDArrayIndex.interval(0,0,true), + INDArray gainView = gradientViewReshaped.get( NDArrayIndex.interval(offset, offset + nOut)); //Already a row vector out.put(GAIN_KEY, gainView); } @@ -196,13 +198,7 @@ public class DefaultParamInitializer extends AbstractParamInitializer { (org.deeplearning4j.nn.conf.layers.FeedForwardLayer) conf; if (initializeParameters) { - if( layerConf.getWeightInit() == null) { - // set a default and set warning - layerConf.setWeightInit(new WeightInitXavier()); - log.warn("Weight Initializer function was not set on layer {} of class {}, it will default to {}", conf.getName(), - conf.getClass().getSimpleName(), WeightInitXavier.class.getSimpleName()); - } - return createWeightMatrix(layerConf.getNIn(), layerConf.getNOut(), layerConf.getWeightInit(), + return createWeightMatrix(layerConf.getNIn(), layerConf.getNOut(), layerConf.getWeightInit(), weightParamView, true); } else { return createWeightMatrix(layerConf.getNIn(), layerConf.getNOut(), null, weightParamView, false); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java index a0a4f059d..6f715cd8f 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java @@ -200,7 +200,7 @@ public class FineTuneConfiguration { bl.setGradientNormalizationThreshold(gradientNormalizationThreshold); } if (updater != null) { - bl.setIUpdater(updater); + bl.setUpdater(updater); } if (biasUpdater != null) { bl.setBiasUpdater(biasUpdater); diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java index 61cd18690..d14513b54 100644 --- a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java @@ -125,7 +125,7 @@ public class NetworkUtils { LayerConfiguration l = net.getLayer(layerNumber).getLayerConfiguration(); if (l instanceof BaseLayerConfiguration) { BaseLayerConfiguration bl = (BaseLayerConfiguration) l; - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); if (u != null && u.hasLearningRate()) { if (newLrSchedule != null) { u.setLrAndSchedule(Double.NaN, newLrSchedule); @@ -207,7 +207,7 @@ public class NetworkUtils { int epoch = net.getEpochCount(); if (l instanceof BaseLayerConfiguration) { BaseLayerConfiguration bl = (BaseLayerConfiguration) l; - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); if (u != null && u.hasLearningRate()) { double d = u.getLearningRate(iter, epoch); if (Double.isNaN(d)) { @@ -247,7 +247,7 @@ public class NetworkUtils { LayerConfiguration l = net.getLayer(layerName).getLayerConfiguration(); if (l instanceof BaseLayerConfiguration) { BaseLayerConfiguration bl = (BaseLayerConfiguration) l; - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); if (u != null && u.hasLearningRate()) { if (newLrSchedule != null) { u.setLrAndSchedule(Double.NaN, newLrSchedule); @@ -329,7 +329,7 @@ public class NetworkUtils { int epoch = net.getComputationGraphConfiguration().getEpochCount(); if (l instanceof BaseLayerConfiguration) { BaseLayerConfiguration bl = (BaseLayerConfiguration) l; - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); if (u != null && u.hasLearningRate()) { double d = u.getLearningRate(iter, epoch); if (Double.isNaN(d)) { diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java index c26a1b7cd..6b92d1a76 100644 --- a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java @@ -210,14 +210,14 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { MultiLayerNetwork netCopy = sparkNet.getNetwork().clone(); netCopy.fit(data); - IUpdater expectedUpdater = ((BaseLayerConfiguration) netCopy.getLayerConfiguration()).getIUpdater(); - double expectedLR = ((Nesterovs)((BaseLayerConfiguration) netCopy.getLayerConfiguration()).getIUpdater()).getLearningRate(); - double expectedMomentum = ((Nesterovs)((BaseLayerConfiguration) netCopy.getLayerConfiguration()).getIUpdater()).getMomentum(); + IUpdater expectedUpdater = ((BaseLayerConfiguration) netCopy.getLayerConfiguration()).getUpdater(); + double expectedLR = ((Nesterovs)((BaseLayerConfiguration) netCopy.getLayerConfiguration()).getUpdater()).getLearningRate(); + double expectedMomentum = ((Nesterovs)((BaseLayerConfiguration) netCopy.getLayerConfiguration()).getUpdater()).getMomentum(); - IUpdater actualUpdater = ((BaseLayerConfiguration) sparkNet.getNetwork().getLayerConfiguration()).getIUpdater(); + IUpdater actualUpdater = ((BaseLayerConfiguration) sparkNet.getNetwork().getLayerConfiguration()).getUpdater(); sparkNet.fit(sparkData); - double actualLR = ((Nesterovs)((BaseLayerConfiguration) sparkNet.getNetwork().getLayerConfiguration()).getIUpdater()).getLearningRate(); - double actualMomentum = ((Nesterovs)((BaseLayerConfiguration) sparkNet.getNetwork().getLayerConfiguration()).getIUpdater()).getMomentum(); + double actualLR = ((Nesterovs)((BaseLayerConfiguration) sparkNet.getNetwork().getLayerConfiguration()).getUpdater()).getLearningRate(); + double actualMomentum = ((Nesterovs)((BaseLayerConfiguration) sparkNet.getNetwork().getLayerConfiguration()).getUpdater()).getMomentum(); assertEquals(expectedUpdater, actualUpdater); assertEquals(expectedLR, actualLR, 0.01); diff --git a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java index 5524bddbc..f0beca2fe 100644 --- a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java @@ -1580,10 +1580,11 @@ public class CudaExecutioner extends DefaultOpExecutioner { zb, (LongPointer) hostZShapeInfo, (LongPointer) AtomicAllocator.getInstance().getPointer(z.shapeInfoDataBuffer(), context), AtomicAllocator.getInstance().getPointer(op.extraArgsDataBuff(z.dataType()), context)); } - - if (nativeOps.lastErrorCode() != 0) - throw new RuntimeException(nativeOps.lastErrorMessage() + " error code: " + nativeOps.lastErrorCode()); - + int errorCode = nativeOps.lastErrorCode(); + if (errorCode != 0) { + throw new RuntimeException( + nativeOps.lastErrorMessage() + " error code: " + errorCode); + } profilingConfigurableHookOut(op, oc, st); return z; diff --git a/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java index 89b2ceef6..36f5f3171 100644 --- a/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java +++ b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java @@ -1189,7 +1189,7 @@ public class TrainModule implements UIModule { throw new RuntimeException(e); } - IUpdater u = bl.getIUpdater(); + IUpdater u = bl.getUpdater(); String us = (u == null ? "" : u.getClass().getSimpleName()); layerInfoRows.add(new String[]{i18N.getMessage("train.model.layerinfotable.layerUpdater"), us});