From 1f4ad083056a194401e640303544bb22be06ece1 Mon Sep 17 00:00:00 2001 From: Robert Altena Date: Thu, 3 Oct 2019 19:35:27 +0900 Subject: [PATCH] refactoring activations. (#8261) Signed-off-by: Robert Altena --- .../nd4j/linalg/activations/Activation.java | 2 +- .../activations/BaseActivationFunction.java | 22 +------------------ .../nd4j/linalg/activations/IActivation.java | 14 +++--------- .../activations/impl/ActivationCube.java | 6 ++--- .../activations/impl/ActivationELU.java | 6 ++--- .../activations/impl/ActivationGELU.java | 6 ++--- .../impl/ActivationHardSigmoid.java | 10 ++++----- .../activations/impl/ActivationHardTanH.java | 10 ++++----- .../activations/impl/ActivationIdentity.java | 4 ++-- .../activations/impl/ActivationLReLU.java | 12 +++++----- .../activations/impl/ActivationPReLU.java | 8 ++----- .../activations/impl/ActivationRReLU.java | 6 ++--- .../impl/ActivationRationalTanh.java | 10 ++++----- .../activations/impl/ActivationReLU.java | 4 ++-- .../activations/impl/ActivationReLU6.java | 6 ++--- .../impl/ActivationRectifiedTanh.java | 10 ++++----- .../activations/impl/ActivationSELU.java | 10 ++++----- .../activations/impl/ActivationSigmoid.java | 9 ++++---- .../activations/impl/ActivationSoftPlus.java | 10 ++++----- .../activations/impl/ActivationSoftSign.java | 10 ++++----- .../activations/impl/ActivationSoftmax.java | 8 +++---- .../activations/impl/ActivationSwish.java | 4 ++-- .../activations/impl/ActivationTanH.java | 4 ++-- .../impl/ActivationThresholdedReLU.java | 6 ++--- 24 files changed, 71 insertions(+), 126 deletions(-) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/Activation.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/Activation.java index 29876606d..86ba0dfef 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/Activation.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/Activation.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java index e8944e759..ccdb119e6 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -32,26 +32,6 @@ public abstract class BaseActivationFunction implements IActivation { return 0; } - @Override - public void setParametersViewArray(INDArray viewArray, boolean initialize) { - //No op - } - - @Override - public INDArray getParametersViewArray() { - return null; - } - - @Override - public void setGradientViewArray(INDArray viewArray) { - //No op - } - - @Override - public INDArray getGradientViewArray() { - return null; - } - protected void assertShape(INDArray in, INDArray epsilon){ if(!in.equalShapes(epsilon)){ throw new IllegalStateException("Shapes must be equal during backprop: in.shape{} = " + Arrays.toString(in.shape()) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java index a334722c0..72a618b1a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -37,8 +37,8 @@ public interface IActivation extends Serializable { * Carry out activation function on the input array (usually known as 'preOut' or 'z') * Implementations must overwrite "in", transform in place and return "in" * Can support separate behaviour during test - * @param in - * @param training + * @param in input array. + * @param training true when training. * @return transformed activation */ INDArray getActivation(INDArray in, boolean training); @@ -59,12 +59,4 @@ public interface IActivation extends Serializable { int numParams(int inputSize); - void setParametersViewArray(INDArray viewArray, boolean initialize); - - INDArray getParametersViewArray(); - - void setGradientViewArray(INDArray viewArray); - - INDArray getGradientViewArray(); - } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java index 766f4a4db..fc4e2217d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -21,17 +21,15 @@ import lombok.Getter; import lombok.NonNull; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.impl.transforms.gradient.CubeBp; import org.nd4j.linalg.api.ops.impl.transforms.same.Cube; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.CubeDerivative; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.primitives.Pair; /** * f(x) = x^3 */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationCube extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java index b714b1f06..95afe8157 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -31,12 +31,12 @@ import org.nd4j.linalg.primitives.Pair; * * alpha defaults to 1, if not specified */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationELU extends BaseActivationFunction { public static final double DEFAULT_ALPHA = 1.0; - private double alpha = DEFAULT_ALPHA; + private double alpha; public ActivationELU() { this(DEFAULT_ALPHA); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java index c26463930..1e16f61a9 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2019 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -32,7 +32,7 @@ import org.nd4j.linalg.primitives.Pair; * * @see GELU */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationGELU extends BaseActivationFunction { @@ -58,7 +58,7 @@ public class ActivationGELU extends BaseActivationFunction { @Override public Pair backprop(INDArray in, INDArray epsilon) { assertShape(in, epsilon); - INDArray dLdz = null; + INDArray dLdz; if (precise) dLdz = Nd4j.getExecutioner().exec(new PreciseGELUDerivative(in, in)); else diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java index 4076b40e3..2cbc95824 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,19 +18,17 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardSigmoidBp; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardSigmoidBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.HardSigmoid; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardSigmoidDerivative; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * f(x) = min(1, max(0, 0.2*x + 0.5)) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationHardSigmoid extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java index f8c405d38..44e732b6a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,21 +18,19 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardTanhBp; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardTanhBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.HardTanh; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardTanhDerivative; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * ⎧ 1, if x > 1 f(x) = ⎨ -1, if x < -1 ⎩ x, otherwise */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationHardTanH extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java index c13002677..d94e87348 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -25,7 +25,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; /** * f(x) = x */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationIdentity extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java index 864f16901..437524314 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,26 +18,24 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.LeakyReLUBp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.LeakyReLUDerivative; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.impl.scalar.LeakyReLU; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.LeakyReLUBp; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * Leaky RELU * f(x) = max(0, x) + alpha * min(0, x) * alpha defaults to 0.01 */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationLReLU extends BaseActivationFunction { public static final double DEFAULT_ALPHA = 0.01; - private double alpha = DEFAULT_ALPHA; + private double alpha; public ActivationLReLU() { this(DEFAULT_ALPHA); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java index 5ff85990c..73321a1c1 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -33,17 +33,13 @@ import org.nd4j.linalg.primitives.Pair; * * @author Max Pumperla */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationPReLU extends BaseActivationFunction { private INDArray alpha; private long[] sharedAxes = null; - public ActivationPReLU(INDArray alpha) { - this.alpha = alpha; - } - public ActivationPReLU(INDArray alpha, long[] sharedAxes) { this.alpha = alpha; this.sharedAxes = sharedAxes; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java index 8d5bb3ddd..478305e76 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -37,7 +37,7 @@ import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; * * Empirical Evaluation of Rectified Activations in Convolutional Network */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @JsonIgnoreProperties({"alpha"}) @Getter public class ActivationRReLU extends BaseActivationFunction { @@ -62,7 +62,7 @@ public class ActivationRReLU extends BaseActivationFunction { @Override public INDArray getActivation(INDArray in, boolean training) { if (training) { - try(MemoryWorkspace ws = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) { + try(MemoryWorkspace ignored = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) { this.alpha = Nd4j.rand(l, u, Nd4j.getRandom(), in.shape()); } INDArray inTimesAlpha = in.mul(alpha); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java index 0e6cc2a51..e67150a93 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,14 +18,12 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.RationalTanhBp; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.RationalTanhBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.RationalTanh; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.RationalTanhDerivative; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * Rational tanh approximation @@ -37,7 +35,7 @@ import org.nd4j.linalg.factory.Nd4j; * * Underlying implementation is in native code */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationRationalTanh extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java index eebee472c..2bafd5472 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -28,7 +28,7 @@ import org.nd4j.linalg.factory.Nd4j; /** * f(x) = max(0, x) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationReLU extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java index 611f2c9ee..103719292 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -20,9 +20,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.impl.scalar.Relu6; -import org.nd4j.linalg.api.ops.impl.scalar.Step; import org.nd4j.linalg.api.ops.impl.transforms.gradient.Relu6Derivative; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.primitives.Pair; @@ -30,7 +28,7 @@ import org.nd4j.linalg.primitives.Pair; /** * f(x) = min(max(input, cutoff), 6) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationReLU6 extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java index ccd4cafe2..252c6d590 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,14 +18,12 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.RectifiedTanhBp; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.RectifiedTanhBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.RectifiedTanh; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.RectifiedTanhDerivative; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * Rectified tanh @@ -34,7 +32,7 @@ import org.nd4j.linalg.factory.Nd4j; * * Underlying implementation is in native code */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationRectifiedTanh extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java index 3eed5ac9c..64c0d5fc0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,19 +18,17 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.SeluBp; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.SeluBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.SELU; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.SELUDerivative; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * https://arxiv.org/pdf/1706.02515.pdf */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationSELU extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java index 72500e677..40674b17a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,18 +18,17 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.SigmoidDerivative; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.SigmoidDerivative; import org.nd4j.linalg.api.ops.impl.transforms.strict.Sigmoid; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * f(x) = 1 / (1 + exp(-x)) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationSigmoid extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java index 0eb7781f2..0306e8ad4 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,19 +18,17 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftPlusBp; -import org.nd4j.linalg.api.ops.impl.transforms.strict.Sigmoid; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftPlusBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.SoftPlus; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * f(x) = log(1+e^x) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationSoftPlus extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java index 3857ff084..31a305cdf 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -18,19 +18,17 @@ package org.nd4j.linalg.activations.impl; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftSignBp; -import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftSignBp; import org.nd4j.linalg.api.ops.impl.transforms.strict.SoftSign; -import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftSignDerivative; import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; /** * f_i(x) = x_i / (1+|x_i|) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationSoftSign extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java index 095c2548a..4cb7bd58b 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -20,8 +20,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import org.nd4j.linalg.activations.BaseActivationFunction; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.CustomOp; -import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.impl.transforms.custom.SoftMax; import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftmaxBp; import org.nd4j.linalg.factory.Nd4j; @@ -31,13 +29,13 @@ import org.nd4j.linalg.primitives.Pair; * f_i(x) = exp(x_i - shift) / sum_j exp(x_j - shift) * where shift = max_i(x_i) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationSoftmax extends BaseActivationFunction { @Override public INDArray getActivation(INDArray in, boolean training) { - Nd4j.getExecutioner().execAndReturn((CustomOp) new SoftMax(in, in)); + Nd4j.getExecutioner().execAndReturn(new SoftMax(in, in)); return in; } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java index f656e6e72..8a1a83ee5 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -28,7 +28,7 @@ import org.nd4j.linalg.primitives.Pair; /** * f(x) = x * sigmoid(x) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationSwish extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java index a30b8e303..b5d580e2a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -28,7 +28,7 @@ import org.nd4j.linalg.factory.Nd4j; /** * f(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationTanH extends BaseActivationFunction { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java index 5736be9ad..71e87e2c7 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ***************************************************************************** * Copyright (c) 2015-2018 Skymind, Inc. * * This program and the accompanying materials are made available under the @@ -31,12 +31,12 @@ import org.nd4j.linalg.primitives.Pair; * * @author Max Pumperla */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Getter public class ActivationThresholdedReLU extends BaseActivationFunction { public static final double DEFAULT_THETA = 1.0; - private double theta = DEFAULT_THETA; + private double theta; public ActivationThresholdedReLU() { this(DEFAULT_THETA);