Brutex
/
cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Layer norm test updates (#187) 

Signed-off-by: Alex Black <blacka101@gmail.com>
master
Alex Black 2019-08-28 13:27:00 +10:00 committed by GitHub
parent 59a6e4e3ae
commit b46f9827b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java
View File

@ -530,7 +530,7 @@ public abstract class DifferentialFunction {
public SDVariable arg(int num){
SDVariable[] args = args();
Preconditions.checkNotNull(args, "Arguments are null for function %s", this.getOwnName());
Preconditions.checkArgument(num >= 0 && num < args.length, "Invalid index: must be 0 to numArgs (0 <= idx < %s)", args.length);
Preconditions.checkArgument(num >= 0 && num < args.length, "Invalid index: must be 0 to numArgs (0 <= idx < %s), got %s", args.length, num);
return args[num];
}

6
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNorm.java
View File

@ -46,6 +46,7 @@ public class LayerNorm extends DynamicCustomOp {
public LayerNorm(@NonNull SameDiff sameDiff, @NonNull SDVariable input, @NonNull SDVariable gain, SDVariable bias, boolean channelsFirst, int... dimensions) {
super(null, sameDiff, wrapFilterNull(input, gain, bias), false);
this.noBias = bias == null;
this.channelsFirst = channelsFirst;
setDimensions(dimensions);
}
@ -56,6 +57,7 @@ public class LayerNorm extends DynamicCustomOp {
public LayerNorm(INDArray input, INDArray gain, INDArray bias, INDArray result, boolean channelsFirst, int... dimensions) {
super("layer_norm", wrapFilterNull(input, gain, bias), wrapOrNull(result));
this.noBias = bias == null;
this.channelsFirst = channelsFirst;
setDimensions(dimensions);
}
@ -115,4 +117,8 @@ public class LayerNorm extends DynamicCustomOp {
return Collections.singletonList(first);
}
@Override
public int numOutputArguments() {
return noBias ? 2 : 3;
}
}

2
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNormBp.java
View File

@ -45,12 +45,14 @@ public class LayerNormBp extends DynamicCustomOp {
public LayerNormBp(@NonNull SameDiff sameDiff, @NonNull SDVariable input, @NonNull SDVariable gain, SDVariable bias, @NonNull SDVariable gradient, boolean channelsFirst, int... dimensions) {
super(null, sameDiff, wrapFilterNull(input, gain, bias, gradient), false);
this.noBias = bias == null;
this.channelsFirst = channelsFirst;
setDimensions(dimensions);
}
public LayerNormBp(@NonNull INDArray input, @NonNull INDArray gain, INDArray bias, @NonNull INDArray grad, @NonNull INDArray dLdx, @NonNull INDArray dLdg, INDArray dLdb, boolean channelsFirst, int... dimensions) {
super("layer_norm_bp", wrapFilterNull(input, gain, bias, grad), wrapFilterNull(dLdx, dLdg, dLdb));
this.noBias = bias == null;
this.channelsFirst = channelsFirst;
setDimensions(dimensions);
}

38
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/LayerOpValidation.java
View File

@ -1112,12 +1112,12 @@ public class LayerOpValidation extends BaseOpValidation {
@Test
public void testLayerNorm() {
final INDArray random = Nd4j.rand(new int[]{10, 4});
final INDArray random = Nd4j.rand(DataType.DOUBLE, 10, 4);
final INDArray standardized = random.ulike();
Nd4j.getExecutioner().exec(new Standardize(random, standardized, 1));
final INDArray gain = Nd4j.rand(new int[]{1, 4});
final INDArray bias = Nd4j.rand(new int[]{1, 4});
final INDArray gain = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray bias = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray res = standardized.mulRowVector(gain).addRowVector(bias);
final INDArray expOut = res.norm1();
@ -1132,7 +1132,7 @@ public class LayerOpValidation extends BaseOpValidation {
String err = OpValidation.validate(new TestCase(sd)
.expectedOutput("out", expOut)
.gradientCheck(true));
assertNull(err, err);
assertNull(err);
}
@Test
@ -1141,9 +1141,9 @@ public class LayerOpValidation extends BaseOpValidation {
int ch = 4;
for(boolean nchw : new boolean[]{true, false}) {
double eps = 0.0;
INDArray x = Nd4j.rand(DataType.FLOAT, nchw ? new long[]{mb, ch, 8, 8} : new long[]{mb, 8, 8, ch});
INDArray gain4d = Nd4j.rand(DataType.FLOAT, nchw ? new long[]{1, ch, 1, 1} : new long[]{1, 1, 1, ch});
INDArray bias4d = Nd4j.rand(DataType.FLOAT, nchw ? new long[]{1, ch, 1, 1} : new long[]{1, 1, 1, ch});
INDArray x = Nd4j.rand(DataType.DOUBLE, nchw ? new long[]{mb, ch, 8, 8} : new long[]{mb, 8, 8, ch});
INDArray gain4d = Nd4j.rand(DataType.DOUBLE, nchw ? new long[]{1, ch, 1, 1} : new long[]{1, 1, 1, ch});
INDArray bias4d = Nd4j.rand(DataType.DOUBLE, nchw ? new long[]{1, ch, 1, 1} : new long[]{1, 1, 1, ch});
INDArray mean = x.mean(true, 1, 2, 3);
INDArray std = Transforms.sqrt(x.var(false,1,2,3).addi(eps)).reshape(mb, 1, 1, 1);
@ -1169,12 +1169,12 @@ public class LayerOpValidation extends BaseOpValidation {
@Test
public void testLayerNormOP() {
final INDArray random = Nd4j.rand(new int[]{10, 4});
final INDArray random = Nd4j.rand(DataType.DOUBLE, 10, 4);
final INDArray standardized = random.ulike();
Nd4j.getExecutioner().exec(new Standardize(random, standardized, 1));
final INDArray gain = Nd4j.rand(new int[]{1, 4});
final INDArray bias = Nd4j.rand(new int[]{1, 4});
final INDArray gain = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray bias = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray res = standardized.mulRowVector(gain).addRowVector(bias);
final INDArray output = Nd4j.zerosLike(res);
@ -1185,11 +1185,11 @@ public class LayerOpValidation extends BaseOpValidation {
@Test
public void testLayerNormNoBias() {
final INDArray random = Nd4j.rand(new int[]{10, 4});
final INDArray random = Nd4j.rand(DataType.DOUBLE, 10, 4);
final INDArray standardized = random.ulike();
Nd4j.getExecutioner().exec(new Standardize(random, standardized, 1));
final INDArray gain = Nd4j.rand(new int[]{1, 4});
final INDArray gain = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray res = standardized.mulRowVector(gain);
final INDArray expOut = res.norm1();
@ -1208,11 +1208,11 @@ public class LayerOpValidation extends BaseOpValidation {
@Test
public void testLayerNormOPNoBias() {
final INDArray random = Nd4j.rand(new int[]{10, 4});
final INDArray random = Nd4j.rand(DataType.DOUBLE, 10, 4);
final INDArray standardized = random.ulike();
Nd4j.getExecutioner().exec(new Standardize(random, standardized, 1));
final INDArray gain = Nd4j.rand(new int[]{1, 4});
final INDArray gain = Nd4j.rand(DataType.DOUBLE,4);
final INDArray res = standardized.mulRowVector(gain);
final INDArray output = Nd4j.zerosLike(res);
@ -1223,7 +1223,7 @@ public class LayerOpValidation extends BaseOpValidation {
@Test
public void testLayerNormNoDeviation() {
final INDArray random = Nd4j.rand(new int[]{10, 4});
final INDArray random = Nd4j.rand(DataType.DOUBLE, 10, 4);
for (int i = 0; i < 4; i++) {
random.putScalar(1,i, 7);
}
@ -1231,8 +1231,8 @@ public class LayerOpValidation extends BaseOpValidation {
final INDArray standardized = random.ulike();
Nd4j.getExecutioner().exec(new Standardize(random, standardized, 1));
final INDArray gain = Nd4j.rand(new int[]{1, 4});
final INDArray bias = Nd4j.rand(new int[]{1, 4});
final INDArray gain = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray bias = Nd4j.rand(DataType.DOUBLE, 4);
final INDArray res = standardized.mulRowVector(gain).addRowVector(bias);
final INDArray expOut = res.norm1();
@ -1332,8 +1332,8 @@ public class LayerOpValidation extends BaseOpValidation {
public void testLayerNormMixedOrders(){
Nd4j.getRandom().setSeed(12345);
INDArray input = Nd4j.rand(DataType.DOUBLE, 3, 8).dup('f');
INDArray gain = Nd4j.rand(DataType.DOUBLE, 1, 8).dup('f');
INDArray bias = Nd4j.rand(DataType.DOUBLE, 1, 8).dup('f');
INDArray gain = Nd4j.rand(DataType.DOUBLE, 8).dup('f');
INDArray bias = Nd4j.rand(DataType.DOUBLE, 8).dup('f');
INDArray outFF = Nd4j.create(DataType.DOUBLE, new long[]{3,8}, 'f');
INDArray outCC = Nd4j.create(DataType.DOUBLE, new long[]{3,8}, 'c');

2
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/opvalidation/TransformOpValidation.java
View File

@ -412,7 +412,7 @@ public class TransformOpValidation extends BaseOpValidation {
.expectedOutput("dp0", expOut[0])
.expectedOutput("dp1", expOut[1])
.gradientCheck(true));
assertNull(err, err);
assertNull(err);
}
@Test