Brutex
/
cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- add parameter alpha to elu and lrelu_bp (#213) 

* - add parameter alpha to elu and lrelu_bp

Signed-off-by: Yurii <yurii@skymind.io>

* - forgot to correct header activations.h

Signed-off-by: Yurii <yurii@skymind.io>
master
Yurii Shyrma 2019-08-31 20:57:39 +03:00 committed by raver119
parent b71c993ded
commit a35926c6e9
11 changed files with 97 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libnd4j/include/loops/legacy_ops.h
View File

@ -116,7 +116,6 @@
#define TRANSFORM_STRICT_OPS \
(3, ELUDerivative), \
(4, TanhDerivative), \
(5, HardTanhDerivative), \
(6, SigmoidDerivative), \
@ -148,7 +147,6 @@
(32, ATan), \
(33, HardTanh), \
(34, SoftSign), \
(35, ELU), \
(36, HardSigmoid), \
(37, RationalTanh) ,\
(38, RectifiedTanh) ,\
@ -211,6 +209,8 @@
(4, ReverseDivide),\
(5, ReverseSubtract),\
(6, MaxPairwise),\
(7, ELU), \
(8, ELUDerivative), \
(13, MinPairwise),\
(14, CopyPws),\
(15, Mod),\

20
libnd4j/include/ops/declarable/generic/activations/elu.cpp
View File

@ -25,12 +25,14 @@
#include <ops/declarable/helpers/legacy_helpers.h>
namespace nd4j {
namespace ops {
CONFIGURABLE_OP_IMPL(elu, 1, 1, true, 0, 0) {
CONFIGURABLE_OP_IMPL(elu, 1, 1, true, -2, 0) {
auto input = INPUT_VARIABLE(0);
auto output = OUTPUT_VARIABLE(0);
input->applyTransform(nd4j::transform::ELU, output, nullptr);
STORE_RESULT(output);
const auto alpha = block.numT() > 0 ? T_ARG(0) : 1.f;
input->applyScalar(nd4j::scalar::ELU, alpha, output);
return Status::OK();
}
@ -41,14 +43,18 @@ namespace nd4j {
->setAllowedOutputTypes(0, {ALL_FLOATS});
}
CONFIGURABLE_OP_IMPL(elu_bp, 2, 1, true, 0, 0) {
CONFIGURABLE_OP_IMPL(elu_bp, 2, 1, true, -2, 0) {
auto input = INPUT_VARIABLE(0);
auto epsilon = INPUT_VARIABLE(1);
auto z = OUTPUT_VARIABLE(0);
auto output = OUTPUT_VARIABLE(0);
const auto alpha = block.numT() > 0 ? T_ARG(0) : 1.f;
// input->applyPairwiseTransform(pairwise::ELUDerivativeE, epsilon, output);
helpers::eluDerivative(block.launchContext(), input, epsilon, output, alpha);
//input->applyPairwiseTransform(pairwise::ELUDerivativeE, epsilon, z, nullptr);
helpers::eluDerivative(block.launchContext(), input, epsilon, z);
return Status::OK();
}

16
libnd4j/include/ops/declarable/generic/activations/lrelu.cpp
View File

@ -25,15 +25,15 @@
#include <ops/declarable/helpers/legacy_helpers.h>
namespace nd4j {
namespace ops {
CONFIGURABLE_OP_IMPL(lrelu, 1, 1, true, 1, 0) {
CONFIGURABLE_OP_IMPL(lrelu, 1, 1, true, -2, 0) {
auto input = INPUT_VARIABLE(0);
auto output = OUTPUT_VARIABLE(0);
float t = block.numT() > 0 ? T_ARG(0) : 0.0f;
float alpha = block.numT() > 0 ? T_ARG(0) : 0.01f;
input->applyScalar(nd4j::scalar::LeakyRELU, t, output);
input->applyScalar(nd4j::scalar::LeakyRELU, alpha, output);
STORE_RESULT(output);
return Status::OK();
}
@ -42,15 +42,17 @@ namespace nd4j {
->setAllowedInputTypes(0, DataType::ANY)
->setAllowedOutputTypes(0, {ALL_FLOATS});
}
CONFIGURABLE_OP_IMPL(lrelu_bp, 2, 1, true, 0, 0) {
CONFIGURABLE_OP_IMPL(lrelu_bp, 2, 1, true, -2, 0) {
auto input = INPUT_VARIABLE(0);
auto epsilon = INPUT_VARIABLE(1);
auto z = OUTPUT_VARIABLE(0);
float alpha = block.numT() > 0 ? T_ARG(0) : 0.01f;
//input->applyPairwiseTransform(pairwise::LRELUDerivativeE, epsilon, z, nullptr);
helpers::leakyReluDerivative(block.launchContext(), input, epsilon, z);
helpers::leakyReluDerivative(block.launchContext(), input, epsilon, z, alpha);
return Status::OK();
}

10
libnd4j/include/ops/declarable/headers/activations.h
View File

@ -82,8 +82,8 @@ namespace nd4j {
* Math is: x < 0 ? alpha * x : x;
*/
#if NOT_EXCLUDED(OP_lrelu)
DECLARE_CONFIGURABLE_OP(lrelu, 1, 1, true, 0, 0);
DECLARE_CONFIGURABLE_OP(lrelu_bp, 2, 1, true, 0, 0);
DECLARE_CONFIGURABLE_OP(lrelu, 1, 1, true, -2, 0);
DECLARE_CONFIGURABLE_OP(lrelu_bp, 2, 1, true, -2, 0);
#endif
/**
@ -91,8 +91,8 @@ namespace nd4j {
* Math is: x >= 0 ? x : exp(x) - 1;
*/
#if NOT_EXCLUDED(OP_elu)
DECLARE_CONFIGURABLE_OP(elu, 1, 1, true, 0, 0);
DECLARE_CONFIGURABLE_OP(elu_bp, 2, 1, true, 0, 0);
DECLARE_CONFIGURABLE_OP(elu, 1, 1, true, -2, 0);
DECLARE_CONFIGURABLE_OP(elu_bp, 2, 1, true, -2, 0);
#endif
/**
@ -157,7 +157,7 @@ namespace nd4j {
/**
* This is Concatenated RELU implementation.
* What happens inside: RELU(Concat((x, -x, {-1})))
*
*
* PLEASE NOTE: Concatenation will double amount of features available in input
*/
#if NOT_EXCLUDED(OP_crelu)

26
libnd4j/include/ops/declarable/helpers/cpu/legacy_helper.cpp
View File

@ -81,29 +81,35 @@ namespace helpers {
}
template <typename T>
static void leakyReluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
auto functor = LAMBDA_TT(x, y){
return x >= (T)0.f? y : T(0.f);
static void leakyReluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output, const float alpha) {
const T alphaT = static_cast<T>(alpha);
auto functor = LAMBDA_TT(x, y, alphaT) {
return x < 0 ? alphaT * y : y;
};
input->applyPairwiseLambda<T>(epsilon, functor, output);
}
void leakyReluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), leakyReluDerivative_, (theFirst, theSecond, theOutput), FLOAT_TYPES);
void leakyReluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput, const float alpha) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), leakyReluDerivative_, (theFirst, theSecond, theOutput, alpha), FLOAT_TYPES);
}
template <typename T>
static void eluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
auto functor = LAMBDA_TT(x, y){
return y * nd4j::math::nd4j_eluderivative<T,T>(x);
static void eluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output, const float alpha) {
const T alphaT = static_cast<T>(alpha);
auto functor = LAMBDA_TT(x, y, alphaT){
return y * nd4j::math::nd4j_eluderivative<T,T>(x, alphaT);
};
input->applyPairwiseLambda<T>(epsilon, functor, output);
}
void eluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), eluDerivative_, (theFirst, theSecond, theOutput), FLOAT_TYPES);
void eluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput, const float alpha) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), eluDerivative_, (theFirst, theSecond, theOutput, alpha), FLOAT_TYPES);
}
template <typename T>

26
libnd4j/include/ops/declarable/helpers/cuda/legacy/relu.cu
View File

@ -66,29 +66,35 @@ namespace nd4j {
}
template <typename T>
linkage void leakyReluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
auto functor = LAMBDA_TT(x, y){
return x >= (T)0.f? y : T(0.f);
linkage void leakyReluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output, const float alpha) {
const T alphaT = static_cast<T>(alpha);
auto functor = LAMBDA_TT(x, y, alphaT) {
return x < 0 ? alphaT * y : y;
};
input->applyPairwiseLambda(epsilon, functor, output);
}
void leakyReluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), leakyReluDerivative_, (theFirst, theSecond, theOutput), FLOAT_TYPES);
void leakyReluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput, const float alpha) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), leakyReluDerivative_, (theFirst, theSecond, theOutput, alpha), FLOAT_TYPES);
}
template <typename T>
linkage void eluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
auto functor = LAMBDA_TT(x, y){
return y * nd4j::math::nd4j_eluderivative<T,T>(x);
linkage void eluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output, const float alpha) {
const T alphaT = static_cast<T>(alpha);
auto functor = LAMBDA_TT(x, y, alphaT){
return y * nd4j::math::nd4j_eluderivative<T,T>(x, alphaT);
};
input->applyPairwiseLambda(epsilon, functor, output);
}
void eluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), eluDerivative_, (theFirst, theSecond, theOutput), FLOAT_TYPES);
void eluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput, const float alpha) {
BUILD_SINGLE_SELECTOR(theFirst->dataType(), eluDerivative_, (theFirst, theSecond, theOutput, alpha), FLOAT_TYPES);
}
template <typename T>

4
libnd4j/include/ops/declarable/helpers/legacy_helpers.h
View File

@ -46,8 +46,8 @@ namespace helpers {
void reluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond);
void reluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);
void relu6Derivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);
void leakyReluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);
void eluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);
void leakyReluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput, const float alpha);
void eluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput, const float alpha);
void seluDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);
void cubeDerivative(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);
void reduceNorm1(nd4j::LaunchContext * context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput);

16
libnd4j/include/ops/ops.h
View File

@ -2271,26 +2271,26 @@ namespace simdOps {
}
};
template <typename X>
template <typename X, typename Y, typename Z>
class ELU {
public:
no_op_exec_special_same
no_op_exec_special_same_cuda
op_def static X op(X d1, X *params) {
return nd4j::math::nd4j_elu<X,X>(d1);
op_def static Z op(X d1, Y d2, Z *params) {
return nd4j::math::nd4j_elu<X,Z>(d1, static_cast<X>(d2));
}
};
template <typename X>
template <typename X, typename Y, typename Z>
class ELUDerivative {
public:
no_op_exec_special_same
no_op_exec_special_same_cuda
op_def static X op(X d1, X *params) {
return nd4j::math::nd4j_eluderivative<X,X>(d1);
op_def static Z op(X d1, Y d2, Z *params) {
return nd4j::math::nd4j_eluderivative<X,Z>(d1, static_cast<X>(d2));
}
};
@ -3716,7 +3716,7 @@ namespace simdOps {
return reduction;
}
op_def static Z op(X d1, X d2, Z *extraParamsRef) {
op_def static Z op(X d1, X d2, Z *extraParamsRef) {
double eps = nd4j::math::nd4j_abs<double>(extraParamsRef[2]);
return static_cast<Z>(!nd4j::math::nd4j_eq<X>(d1, d2, eps));
}
@ -4540,4 +4540,4 @@ namespace simdOps {
}
#endif

20
libnd4j/include/templatemath.h
View File

@ -130,13 +130,12 @@ namespace nd4j {
}
template<typename T, typename Z>
math_def inline Z nd4j_elu(T val) {
if (val >= (T) 0.f) return val;
else return nd4j_exp<T, Z>(val) - (Z) 1.0f;
//return val >= 0.0 ? val : (nd4j_exp<T>(val) - 1.0);
math_def inline Z nd4j_elu(T val, T alpha) {
if (val >= (T) 0.f)
return val;
return static_cast<Z>(alpha) * (nd4j_exp<T, Z>(val) - static_cast<Z>(1.0f));
}
template<typename T, typename Z>
math_def inline Z nd4j_leakyrelu(T val,T alpha) {
if (val < (T) 0.0f)
@ -145,13 +144,14 @@ namespace nd4j {
return val;
}
template<typename T, typename Z>
math_def inline Z nd4j_eluderivative(T val) {
if (val >= (T) 0.0f) return (Z) 1.0f;
else return nd4j_exp<T, Z>(val);
math_def inline Z nd4j_eluderivative(T val, T alpha) {
if (val >= static_cast<T>(0.0f))
return static_cast<Z>(1.0f);
return static_cast<Z>(alpha) * nd4j_exp<T, Z>(val);
//return val >= 0.0 ? 1.0 : nd4j_exp(val);
}
template<typename T, typename Z>
math_def inline Z nd4j_sin(T val);
@ -283,7 +283,7 @@ namespace nd4j {
#ifdef NATIVE_HALFS
if (value < (float16) 0.f) {
return float16(__hneg(value.data));
} else
} else
return value;
#else
return (float16) fabsf((float) value);

51
libnd4j/tests_cpu/layers_tests/DeclarableOpsTests3.cpp
View File

@ -2794,53 +2794,42 @@ TEST_F(DeclarableOpsTests3, svd_test11) {
TEST_F(DeclarableOpsTests3, elu_test1) {
auto x = NDArrayFactory::create<double>('c', {3,3}, {0.1, .2, .3, -.4,-.5,-.6, .7, .8, .9});
// auto expS = NDArrayFactory::create<double>('c', {3});
// auto expU = NDArrayFactory::create<double>('c', {3,3});
auto exp = NDArrayFactory::create<double>('c', {3,3}, {.1, .2, .3, -0.32968, -0.393469, -0.451188, .7, .8, .9});
auto exp = NDArrayFactory::create<double>('c', {3,3}, {.1, .2, .3, 0.5*-0.32968, 0.5*-0.393469, 0.5*-0.451188, .7, .8, .9});
nd4j::ops::elu op;
auto results = op.execute({&x}, {}, {});
auto results = op.execute({&x}, {0.5}, {});
ASSERT_EQ(ND4J_STATUS_OK, results->status());
auto s = results->at(0);
// auto u = results->at(1);
// auto v = results->at(2);
// s->printIndexedBuffer("ELU");
ASSERT_TRUE(exp.equalsTo(s));
delete results;
}
///////////////////////////////////////////////////////////////////
TEST_F(DeclarableOpsTests3, elu_test2) {
TEST_F(DeclarableOpsTests3, elu_bp_test1) {
auto x = NDArrayFactory::create<double>('c', {3, 3}, {0.1, .2, .3, -.4, -.5, -.6, .7, .8, .9});
auto eps = NDArrayFactory::create<double>('c', {3,3});
eps.assign(2.);
// auto expU = NDArrayFactory::create<double>('c', {3,3});
auto exp = NDArrayFactory::create<double>('c', {3, 3}, {2, 2, 2, 1.34064, 1.213061, 1.097623, 2, 2, 2});
auto x = NDArrayFactory::create<double>('c', {3, 3}, {0.1, .2, .3, -.4, -.5, -.6, .7, .8, .9});
auto eps = NDArrayFactory::create<double>('c', {3,3});
eps.assign(2.);
auto exp = NDArrayFactory::create<double>('c', {3, 3}, {2, 2, 2, 0.5*1.34064, 0.5*1.213061, 0.5*1.097623, 2, 2, 2});
nd4j::ops::elu_bp op;
auto results = op.execute({ &x, &eps }, {}, {});
nd4j::ops::elu_bp op;
auto results = op.execute({ &x, &eps }, {0.5}, {});
ASSERT_EQ(ND4J_STATUS_OK, results->status());
ASSERT_EQ(ND4J_STATUS_OK, results->status());
auto s = results->at(0);
// auto u = results->at(1);
// auto v = results->at(2);
// s->printIndexedBuffer("ELU_BP");
ASSERT_TRUE(exp.equalsTo(s));
auto s = results->at(0);
ASSERT_TRUE(exp.equalsTo(s));
delete results;
delete results;
}
///////////////////////////////////////////////////////////////////
TEST_F(DeclarableOpsTests3, lrelu_test1) {
auto x = NDArrayFactory::create<double>('c', {3,3}, {1, 2, 3, -4,-5,-6, 7, 8, 9});
// auto expS = NDArrayFactory::create<double>('c', {3});
// auto expU = NDArrayFactory::create<double>('c', {3,3});
auto exp = NDArrayFactory::create<double>('c', {3,3}, {1, 2, 3, -0.8, -1., -1.2, 7, 8, 9});
nd4j::ops::lrelu op;
@ -2849,20 +2838,16 @@ TEST_F(DeclarableOpsTests3, lrelu_test1) {
ASSERT_EQ(ND4J_STATUS_OK, results->status());
auto s = results->at(0);
// auto u = results->at(1);
// auto v = results->at(2);
// s->printIndexedBuffer("LRELU");
ASSERT_TRUE(exp.equalsTo(s));
delete results;
}
TEST_F(DeclarableOpsTests3, lrelu_test2) {
TEST_F(DeclarableOpsTests3, lrelu_bp_test1) {
auto x = NDArrayFactory::create<double>('c', {3,3}, {1, 2, 3, -4,-5,-6, 7, 8, 9});
// auto expS = NDArrayFactory::create<double>('c', {3});
auto eps = NDArrayFactory::create<double>('c', {3,3}, {2,2,2,2,2,2,2, 2,2});
auto exp = NDArrayFactory::create<double>('c', {3,3}, {2, 2, 2, 0, 0, 0, 2, 2, 2});
auto exp = NDArrayFactory::create<double>('c', {3,3}, {2, 2, 2, 0.4, 0.4, 0.4, 2, 2, 2});
nd4j::ops::lrelu_bp op;
auto results = op.execute({&x, &eps}, {0.2}, {});
@ -2870,9 +2855,6 @@ TEST_F(DeclarableOpsTests3, lrelu_test2) {
ASSERT_EQ(ND4J_STATUS_OK, results->status());
auto s = results->at(0);
// auto u = results->at(1);
// auto v = results->at(2);
// s->printIndexedBuffer("LRELU_BP");
ASSERT_TRUE(exp.equalsTo(s));
delete results;
@ -2882,8 +2864,6 @@ TEST_F(DeclarableOpsTests3, lrelu_test2) {
TEST_F(DeclarableOpsTests3, selu_test1) {
auto x = NDArrayFactory::create<double>('c', {3,3}, {1, 2, 3, -4,-5,-6, 7, 8, 9});
// auto expS = NDArrayFactory::create<double>('c', {3});
// auto expU = NDArrayFactory::create<double>('c', {3,3});
auto exp = NDArrayFactory::create<double>('c', {3,3}, {1.050701, 2.101402, 3.152103, -1.725899, -1.746253, -1.753742, 7.354907, 8.405608, 9.456309});
nd4j::ops::selu op;
@ -2892,7 +2872,6 @@ TEST_F(DeclarableOpsTests3, selu_test1) {
ASSERT_EQ(ND4J_STATUS_OK, results->status());
auto s = results->at(0);
// s->printIndexedBuffer("SELU");
ASSERT_TRUE(exp.equalsTo(s));
delete results;

2
libnd4j/tests_cpu/layers_tests/DeclarableOpsTests5.cpp
View File

@ -2761,7 +2761,7 @@ TEST_F(DeclarableOpsTests5, ELU_1) {
auto exp = NDArrayFactory::create<double>('c', {2, 2, 2}, { -0.63212055, 2. , 1.5, -0.753403, 1., 2., 2., 1.});
auto res = NDArrayFactory::create<double>('c', {2, 2, 2});
input.applyTransform(transform::ELU, &res);
input.applyScalar(nd4j::scalar::ELU, 1.f, &res);
ASSERT_TRUE(res.equalsTo(&exp));
}