Brutex
/
cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SameDiff If, While, and Misc changes (#52) 

* softmax and logSoftmax w/ dimension

Signed-off-by: Ryan Nett <rnett@skymind.io>

* start of while

Signed-off-by: Ryan Nett <rnett@skymind.io>

* if, start of javadocs

Signed-off-by: Ryan Nett <rnett@skymind.io>

* while foreward pass working, backprop WIP

Signed-off-by: Ryan Nett <rnett@skymind.io>

* no backprop

Signed-off-by: Ryan Nett <rnett@skymind.io>

* Tensorflow style if/while (& tests), name scope fixes (and test), argument interceptor (for if/while), use '_' in op names instead of ':'

Signed-off-by: Ryan Nett <rnett@skymind.io>

* javadoc

Signed-off-by: Ryan Nett <rnett@skymind.io>

* many fixes

Signed-off-by: Ryan Nett <rnett@skymind.io>

* many fixes

Signed-off-by: Ryan Nett <rnett@skymind.io>

* Some fixes

Signed-off-by: Ryan Nett <rnett@skymind.io>

* cleanup if condition doesn't return boolean

Signed-off-by: Ryan Nett <rnett@skymind.io>

* serialization fix

Signed-off-by: Ryan Nett <rnett@skymind.io>

* use constants instead of magic numbers

Signed-off-by: Ryan Nett <rnett@skymind.io>
master
Ryan Nett 2019-07-11 21:08:15 -07:00 committed by AlexDBlack
parent 2d991f5445
commit daf3950d8d
23 changed files with 1662 additions and 379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java
View File

@ -451,6 +451,17 @@ public abstract class DifferentialFunction {
}
}
public void replaceArg(int i, SDVariable newArg){
if(sameDiff != null){
sameDiff.replaceArgFor(i, newArg, this);
if(args()[i].isPlaceHolder() && !newArg.isPlaceHolder()){
sameDiff.removePropertyToResolve(this, args()[i].getVarName());
} else if(!args()[i].isPlaceHolder() && newArg.isPlaceHolder()){
sameDiff.addPropertyToResolve(this, newArg.getVarName());
}
}
}
/**
* Return the output variables for this differential function.
@ -652,9 +663,9 @@ public abstract class DifferentialFunction {
scope = "";
else
scope = scope + "/";
String varName = scope + sameDiff.generateNewVarName(opName(),argIndex);
String varName = scope + sameDiff.generateNewVarName(opName(),argIndex).replace(":", "_");
while(sameDiff.functionExists(varName)) {
varName = scope + sameDiff.generateNewVarName(opName(), argIndex);
varName = scope + sameDiff.generateNewVarName(opName(), argIndex).replace(":", "_");
argIndex++;
}

369
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunctionFactory.java
View File

@ -16,6 +16,11 @@
package org.nd4j.autodiff.functions;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
import lombok.NonNull;
import lombok.val;
@ -30,36 +35,183 @@ import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.NoOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BiasAdd;
import org.nd4j.linalg.api.ops.impl.broadcast.BiasAddGrad;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Enter;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Exit;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Merge;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.NextIteration;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Switch;
import org.nd4j.linalg.api.ops.impl.image.ExtractImagePatches;
import org.nd4j.linalg.api.ops.impl.indexaccum.*;
import org.nd4j.linalg.api.ops.impl.indexaccum.FirstIndex;
import org.nd4j.linalg.api.ops.impl.indexaccum.IAMax;
import org.nd4j.linalg.api.ops.impl.indexaccum.IAMin;
import org.nd4j.linalg.api.ops.impl.indexaccum.IMax;
import org.nd4j.linalg.api.ops.impl.indexaccum.IMin;
import org.nd4j.linalg.api.ops.impl.indexaccum.LastIndex;
import org.nd4j.linalg.api.ops.impl.layers.ExternalErrorsFunction;
import org.nd4j.linalg.api.ops.impl.layers.convolution.*;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.*;
import org.nd4j.linalg.api.ops.impl.loss.*;
import org.nd4j.linalg.api.ops.impl.loss.bp.*;
import org.nd4j.linalg.api.ops.impl.reduce.*;
import org.nd4j.linalg.api.ops.impl.layers.convolution.AvgPooling2D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.BatchNorm;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Col2Im;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Conv1D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Conv2D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Conv3D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv2D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv3D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.DeConv3DDerivative;
import org.nd4j.linalg.api.ops.impl.layers.convolution.DepthToSpace;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Im2col;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Im2colBp;
import org.nd4j.linalg.api.ops.impl.layers.convolution.LocalResponseNormalization;
import org.nd4j.linalg.api.ops.impl.layers.convolution.MaxPooling2D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Pooling3D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.SConv2D;
import org.nd4j.linalg.api.ops.impl.layers.convolution.SpaceToDepth;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Upsampling2d;
import org.nd4j.linalg.api.ops.impl.layers.convolution.Upsampling2dDerivative;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv1DConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv2DConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv3DConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.DeConv2DConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.DeConv3DConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.LocalResponseNormalizationConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Pooling2DConfig;
import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Pooling3DConfig;
import org.nd4j.linalg.api.ops.impl.loss.AbsoluteDifferenceLoss;
import org.nd4j.linalg.api.ops.impl.loss.CosineDistanceLoss;
import org.nd4j.linalg.api.ops.impl.loss.HingeLoss;
import org.nd4j.linalg.api.ops.impl.loss.HuberLoss;
import org.nd4j.linalg.api.ops.impl.loss.L2Loss;
import org.nd4j.linalg.api.ops.impl.loss.LogLoss;
import org.nd4j.linalg.api.ops.impl.loss.LogPoissonLoss;
import org.nd4j.linalg.api.ops.impl.loss.MeanPairwiseSquaredErrorLoss;
import org.nd4j.linalg.api.ops.impl.loss.MeanSquaredErrorLoss;
import org.nd4j.linalg.api.ops.impl.loss.SigmoidCrossEntropyLoss;
import org.nd4j.linalg.api.ops.impl.loss.SoftmaxCrossEntropyLoss;
import org.nd4j.linalg.api.ops.impl.loss.SoftmaxCrossEntropyWithLogitsLoss;
import org.nd4j.linalg.api.ops.impl.loss.SparseSoftmaxCrossEntropyLossWithLogits;
import org.nd4j.linalg.api.ops.impl.loss.WeightedCrossEntropyLoss;
import org.nd4j.linalg.api.ops.impl.loss.bp.AbsoluteDifferenceLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.CosineDistanceLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.HingeLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.HuberLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.LogLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.LogPoissonLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.MeanPairwiseSquaredErrorLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.MeanSquaredErrorLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.SigmoidCrossEntropyLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.SoftmaxCrossEntropyLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.SoftmaxCrossEntropyWithLogitsLossBp;
import org.nd4j.linalg.api.ops.impl.loss.bp.SparseSoftmaxCrossEntropyLossWithLogitsBp;
import org.nd4j.linalg.api.ops.impl.reduce.Mmul;
import org.nd4j.linalg.api.ops.impl.reduce.MmulBp;
import org.nd4j.linalg.api.ops.impl.reduce.Moments;
import org.nd4j.linalg.api.ops.impl.reduce.NormalizeMoments;
import org.nd4j.linalg.api.ops.impl.reduce.TensorMmul;
import org.nd4j.linalg.api.ops.impl.reduce.ZeroFraction;
import org.nd4j.linalg.api.ops.impl.reduce.bool.All;
import org.nd4j.linalg.api.ops.impl.reduce.bool.Any;
import org.nd4j.linalg.api.ops.impl.reduce.bp.*;
import org.nd4j.linalg.api.ops.impl.reduce.bp.CumProdBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.CumSumBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.DotBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.MaxBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.MeanBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.MinBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.Norm1Bp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.Norm2Bp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.NormMaxBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.ProdBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.SquaredNormBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.StandardDeviationBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.SumBp;
import org.nd4j.linalg.api.ops.impl.reduce.bp.VarianceBp;
import org.nd4j.linalg.api.ops.impl.reduce.custom.BatchMmul;
import org.nd4j.linalg.api.ops.impl.reduce.custom.LogSumExp;
import org.nd4j.linalg.api.ops.impl.reduce.floating.*;
import org.nd4j.linalg.api.ops.impl.reduce.floating.AMean;
import org.nd4j.linalg.api.ops.impl.reduce.floating.Entropy;
import org.nd4j.linalg.api.ops.impl.reduce.floating.LogEntropy;
import org.nd4j.linalg.api.ops.impl.reduce.floating.Mean;
import org.nd4j.linalg.api.ops.impl.reduce.floating.Norm1;
import org.nd4j.linalg.api.ops.impl.reduce.floating.Norm2;
import org.nd4j.linalg.api.ops.impl.reduce.floating.NormMax;
import org.nd4j.linalg.api.ops.impl.reduce.floating.ShannonEntropy;
import org.nd4j.linalg.api.ops.impl.reduce.floating.SquaredNorm;
import org.nd4j.linalg.api.ops.impl.reduce.longer.CountNonZero;
import org.nd4j.linalg.api.ops.impl.reduce.longer.CountZero;
import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition;
import org.nd4j.linalg.api.ops.impl.reduce.same.AMax;
import org.nd4j.linalg.api.ops.impl.reduce.same.AMin;
import org.nd4j.linalg.api.ops.impl.reduce.same.ASum;
import org.nd4j.linalg.api.ops.impl.reduce.same.Max;
import org.nd4j.linalg.api.ops.impl.reduce.same.Min;
import org.nd4j.linalg.api.ops.impl.reduce.same.*;
import org.nd4j.linalg.api.ops.impl.reduce3.*;
import org.nd4j.linalg.api.ops.impl.reduce.same.Prod;
import org.nd4j.linalg.api.ops.impl.reduce.same.Sum;
import org.nd4j.linalg.api.ops.impl.reduce3.CosineDistance;
import org.nd4j.linalg.api.ops.impl.reduce3.CosineSimilarity;
import org.nd4j.linalg.api.ops.impl.reduce3.Dot;
import org.nd4j.linalg.api.ops.impl.reduce3.EuclideanDistance;
import org.nd4j.linalg.api.ops.impl.reduce3.HammingDistance;
import org.nd4j.linalg.api.ops.impl.reduce3.JaccardDistance;
import org.nd4j.linalg.api.ops.impl.reduce3.ManhattanDistance;
import org.nd4j.linalg.api.ops.impl.scalar.LeakyReLU;
import org.nd4j.linalg.api.ops.impl.scalar.LogX;
import org.nd4j.linalg.api.ops.impl.scalar.Pow;
import org.nd4j.linalg.api.ops.impl.scalar.*;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.*;
import org.nd4j.linalg.api.ops.impl.scatter.*;
import org.nd4j.linalg.api.ops.impl.shape.*;
import org.nd4j.linalg.api.ops.impl.scalar.PowDerivative;
import org.nd4j.linalg.api.ops.impl.scalar.RectifiedLinear;
import org.nd4j.linalg.api.ops.impl.scalar.Relu6;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarAdd;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarDivision;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarFMod;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarMax;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarMin;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarMultiplication;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarReverseDivision;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarReverseSubtraction;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarSet;
import org.nd4j.linalg.api.ops.impl.scalar.ScalarSubtraction;
import org.nd4j.linalg.api.ops.impl.scalar.Step;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.ScalarEquals;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.ScalarGreaterThan;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.ScalarGreaterThanOrEqual;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.ScalarLessThan;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.ScalarLessThanOrEqual;
import org.nd4j.linalg.api.ops.impl.scalar.comparison.ScalarNotEquals;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterAdd;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterDiv;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterMax;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterMin;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterMul;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterSub;
import org.nd4j.linalg.api.ops.impl.scatter.ScatterUpdate;
import org.nd4j.linalg.api.ops.impl.shape.Broadcast;
import org.nd4j.linalg.api.ops.impl.shape.Concat;
import org.nd4j.linalg.api.ops.impl.shape.ConfusionMatrix;
import org.nd4j.linalg.api.ops.impl.shape.Cross;
import org.nd4j.linalg.api.ops.impl.shape.Diag;
import org.nd4j.linalg.api.ops.impl.shape.DiagPart;
import org.nd4j.linalg.api.ops.impl.shape.ExpandDims;
import org.nd4j.linalg.api.ops.impl.shape.Gather;
import org.nd4j.linalg.api.ops.impl.shape.GatherNd;
import org.nd4j.linalg.api.ops.impl.shape.MergeAvg;
import org.nd4j.linalg.api.ops.impl.shape.MergeMax;
import org.nd4j.linalg.api.ops.impl.shape.MeshGrid;
import org.nd4j.linalg.api.ops.impl.shape.OneHot;
import org.nd4j.linalg.api.ops.impl.shape.OnesLike;
import org.nd4j.linalg.api.ops.impl.shape.ParallelStack;
import org.nd4j.linalg.api.ops.impl.shape.Permute;
import org.nd4j.linalg.api.ops.impl.shape.Rank;
import org.nd4j.linalg.api.ops.impl.shape.ReductionShape;
import org.nd4j.linalg.api.ops.impl.shape.Repeat;
import org.nd4j.linalg.api.ops.impl.shape.Reshape;
import org.nd4j.linalg.api.ops.impl.shape.SequenceMask;
import org.nd4j.linalg.api.ops.impl.shape.Size;
import org.nd4j.linalg.api.ops.impl.shape.SizeAt;
import org.nd4j.linalg.api.ops.impl.shape.Slice;
import org.nd4j.linalg.api.ops.impl.shape.Squeeze;
import org.nd4j.linalg.api.ops.impl.shape.Stack;
import org.nd4j.linalg.api.ops.impl.shape.StridedSlice;
import org.nd4j.linalg.api.ops.impl.shape.Tile;
import org.nd4j.linalg.api.ops.impl.shape.Transpose;
import org.nd4j.linalg.api.ops.impl.shape.Unstack;
import org.nd4j.linalg.api.ops.impl.shape.ZerosLike;
import org.nd4j.linalg.api.ops.impl.shape.bp.SliceBp;
import org.nd4j.linalg.api.ops.impl.shape.bp.StridedSliceBp;
import org.nd4j.linalg.api.ops.impl.shape.bp.TileBp;
@ -77,37 +229,165 @@ import org.nd4j.linalg.api.ops.impl.transforms.clip.ClipByNorm;
import org.nd4j.linalg.api.ops.impl.transforms.clip.ClipByValue;
import org.nd4j.linalg.api.ops.impl.transforms.comparison.CompareAndReplace;
import org.nd4j.linalg.api.ops.impl.transforms.comparison.CompareAndSet;
import org.nd4j.linalg.api.ops.impl.transforms.custom.*;
import org.nd4j.linalg.api.ops.impl.transforms.custom.segment.*;
import org.nd4j.linalg.api.ops.impl.transforms.custom.ATan2;
import org.nd4j.linalg.api.ops.impl.transforms.custom.Assign;
import org.nd4j.linalg.api.ops.impl.transforms.custom.BatchToSpace;
import org.nd4j.linalg.api.ops.impl.transforms.custom.CumProd;
import org.nd4j.linalg.api.ops.impl.transforms.custom.CumSum;
import org.nd4j.linalg.api.ops.impl.transforms.custom.Dilation2D;
import org.nd4j.linalg.api.ops.impl.transforms.custom.DotProductAttention;
import org.nd4j.linalg.api.ops.impl.transforms.custom.DotProductAttentionBp;
import org.nd4j.linalg.api.ops.impl.transforms.custom.DynamicPartition;
import org.nd4j.linalg.api.ops.impl.transforms.custom.DynamicStitch;
import org.nd4j.linalg.api.ops.impl.transforms.custom.EqualTo;
import org.nd4j.linalg.api.ops.impl.transforms.custom.Fill;
import org.nd4j.linalg.api.ops.impl.transforms.custom.GreaterThan;
import org.nd4j.linalg.api.ops.impl.transforms.custom.GreaterThanOrEqual;
import org.nd4j.linalg.api.ops.impl.transforms.custom.InvertPermutation;
import org.nd4j.linalg.api.ops.impl.transforms.custom.IsNonDecreasing;
import org.nd4j.linalg.api.ops.impl.transforms.custom.IsNumericTensor;
import org.nd4j.linalg.api.ops.impl.transforms.custom.IsStrictlyIncreasing;
import org.nd4j.linalg.api.ops.impl.transforms.custom.LayerNorm;
import org.nd4j.linalg.api.ops.impl.transforms.custom.LayerNormBp;
import org.nd4j.linalg.api.ops.impl.transforms.custom.LessThan;
import org.nd4j.linalg.api.ops.impl.transforms.custom.LessThanOrEqual;
import org.nd4j.linalg.api.ops.impl.transforms.custom.ListDiff;
import org.nd4j.linalg.api.ops.impl.transforms.custom.LogSoftMax;
import org.nd4j.linalg.api.ops.impl.transforms.custom.MatrixDeterminant;
import org.nd4j.linalg.api.ops.impl.transforms.custom.MatrixInverse;
import org.nd4j.linalg.api.ops.impl.transforms.custom.MatrixSetDiag;
import org.nd4j.linalg.api.ops.impl.transforms.custom.MultiHeadDotProductAttention;
import org.nd4j.linalg.api.ops.impl.transforms.custom.MultiHeadDotProductAttentionBp;
import org.nd4j.linalg.api.ops.impl.transforms.custom.NotEqualTo;
import org.nd4j.linalg.api.ops.impl.transforms.custom.Reverse;
import org.nd4j.linalg.api.ops.impl.transforms.custom.ReverseSequence;
import org.nd4j.linalg.api.ops.impl.transforms.custom.SoftMax;
import org.nd4j.linalg.api.ops.impl.transforms.custom.SpaceToBatch;
import org.nd4j.linalg.api.ops.impl.transforms.custom.Standardize;
import org.nd4j.linalg.api.ops.impl.transforms.custom.StandardizeBp;
import org.nd4j.linalg.api.ops.impl.transforms.custom.Trace;
import org.nd4j.linalg.api.ops.impl.transforms.custom.XwPlusB;
import org.nd4j.linalg.api.ops.impl.transforms.custom.segment.SegmentMax;
import org.nd4j.linalg.api.ops.impl.transforms.custom.segment.SegmentMean;
import org.nd4j.linalg.api.ops.impl.transforms.custom.segment.SegmentMin;
import org.nd4j.linalg.api.ops.impl.transforms.custom.segment.SegmentProd;
import org.nd4j.linalg.api.ops.impl.transforms.custom.segment.SegmentSum;
import org.nd4j.linalg.api.ops.impl.transforms.dtype.Cast;
import org.nd4j.linalg.api.ops.impl.transforms.floating.RSqrt;
import org.nd4j.linalg.api.ops.impl.transforms.floating.Sqrt;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.CubeDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.DynamicPartitionBp;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.ELUDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.GradientBackwardsMarker;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.HardTanhDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.LeakyReLUDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.LogSoftMaxDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.RationalTanhDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.RectifiedTanhDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.Relu6Derivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.SELUDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.SigmoidDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.*;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.*;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.*;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftSignDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.SoftmaxBp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.AddOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.DivOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.FloorDivOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.FloorModOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.MergeAddOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.MulOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.RDivOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.RSubOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.SquaredDifferenceOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.SubOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.TruncateDivOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.AddBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.DivBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.FloorDivBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.FloorModBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.MulBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.RDivBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.RSubBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.bp.SubBpOp;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.bool.And;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.bool.Or;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.bool.Xor;
import org.nd4j.linalg.api.ops.impl.transforms.same.*;
import org.nd4j.linalg.api.ops.impl.transforms.segment.*;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.*;
import org.nd4j.linalg.api.ops.impl.transforms.strict.*;
import org.nd4j.linalg.api.ops.impl.transforms.same.Abs;
import org.nd4j.linalg.api.ops.impl.transforms.same.Ceil;
import org.nd4j.linalg.api.ops.impl.transforms.same.Cube;
import org.nd4j.linalg.api.ops.impl.transforms.same.Floor;
import org.nd4j.linalg.api.ops.impl.transforms.same.Identity;
import org.nd4j.linalg.api.ops.impl.transforms.same.Negative;
import org.nd4j.linalg.api.ops.impl.transforms.same.Reciprocal;
import org.nd4j.linalg.api.ops.impl.transforms.same.Round;
import org.nd4j.linalg.api.ops.impl.transforms.same.Sign;
import org.nd4j.linalg.api.ops.impl.transforms.same.Square;
import org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentMax;
import org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentMean;
import org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentMin;
import org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentProd;
import org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentSqrtN;
import org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentSum;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.SegmentMaxBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.SegmentMeanBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.SegmentMinBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.SegmentProdBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.SegmentSumBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentMaxBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentMeanBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentMinBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentProdBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentSqrtNBp;
import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentSumBp;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ACos;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ACosh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ASin;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ASinh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ATan;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ATanh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Cos;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Cosh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.ELU;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Erf;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Erfc;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Exp;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Expm1;
import org.nd4j.linalg.api.ops.impl.transforms.strict.GELU;
import org.nd4j.linalg.api.ops.impl.transforms.strict.GELUDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.strict.HardSigmoid;
import org.nd4j.linalg.api.ops.impl.transforms.strict.HardTanh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Log;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Log1p;
import org.nd4j.linalg.api.ops.impl.transforms.strict.LogSigmoid;
import org.nd4j.linalg.api.ops.impl.transforms.strict.PreciseGELU;
import org.nd4j.linalg.api.ops.impl.transforms.strict.PreciseGELUDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.strict.RationalTanh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.RectifiedTanh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.SELU;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Sigmoid;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Sin;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Sinh;
import org.nd4j.linalg.api.ops.impl.transforms.strict.SoftPlus;
import org.nd4j.linalg.api.ops.impl.transforms.strict.SoftSign;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Swish;
import org.nd4j.linalg.api.ops.impl.transforms.strict.SwishDerivative;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Tan;
import org.nd4j.linalg.api.ops.impl.transforms.strict.Tanh;
import org.nd4j.linalg.api.ops.random.custom.DistributionUniform;
import org.nd4j.linalg.api.ops.random.custom.RandomBernoulli;
import org.nd4j.linalg.api.ops.random.custom.RandomExponential;
import org.nd4j.linalg.api.ops.random.custom.RandomNormal;
import org.nd4j.linalg.api.ops.random.impl.*;
import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution;
import org.nd4j.linalg.api.ops.random.impl.BinomialDistribution;
import org.nd4j.linalg.api.ops.random.impl.DropOutInverted;
import org.nd4j.linalg.api.ops.random.impl.GaussianDistribution;
import org.nd4j.linalg.api.ops.random.impl.LogNormalDistribution;
import org.nd4j.linalg.api.ops.random.impl.Range;
import org.nd4j.linalg.api.ops.random.impl.TruncatedNormalDistribution;
import org.nd4j.linalg.api.ops.random.impl.UniformDistribution;
import org.nd4j.linalg.api.shape.Shape;
import org.nd4j.linalg.indexing.conditions.Condition;
import org.nd4j.linalg.util.ArrayUtil;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
*
*/
@ -1611,11 +1891,24 @@ public class DifferentialFunctionFactory {
}
public SDVariable logSoftmax(SDVariable i_v, int dimension) {
validateDifferentialFunctionsameDiff(i_v);
return new LogSoftMax(sameDiff(), i_v, dimension).outputVariable();
}
public SDVariable logSoftmaxDerivative(SDVariable arg, SDVariable wrt) {
validateDifferentialFunctionsameDiff(arg);
return new LogSoftMaxDerivative(sameDiff(), arg, wrt).outputVariable();
}
public SDVariable logSoftmaxDerivative(SDVariable arg, SDVariable wrt, int dimension) {
validateDifferentialFunctionsameDiff(arg);
return new LogSoftMaxDerivative(sameDiff(), arg, wrt, dimension).outputVariable();
}
public SDVariable logSumExp(SDVariable arg, boolean keepDims, int... dimension) {
return new LogSumExp(sameDiff(), arg, keepDims, dimension).outputVariable();
}
@ -2296,6 +2589,22 @@ public class DifferentialFunctionFactory {
return tile(func, ArrayUtil.toInts(input.getShape()));
}
public SDVariable enter(SDVariable x, String frameName){
return new Enter(sameDiff, frameName, x).outputVariable();
}
public SDVariable enter(SDVariable x, String frameName, boolean isConstant){
return new Enter(sameDiff, frameName, x, isConstant).outputVariable();
}
public SDVariable exit(SDVariable x){
return new Exit(sameDiff, x).outputVariable();
}
public SDVariable nextIteration(SDVariable x){
return new NextIteration(sameDiff, x).outputVariable();
}
public String toString() {
return "DifferentialFunctionFactory{methodNames=" + methodNames + "}";

30
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ArgumentInterceptor.java Normal file
View File

@ -0,0 +1,30 @@
/*******************************************************************************
* Copyright (c) 2015-2019 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.nd4j.autodiff.samediff;
/**
* Internal interface used to apply a transform to any arguments used within a certain block
*
* Intended for internal use only.
*
* Managed with {@link SameDiff#addArgumentInterceptor(ArgumentInterceptor)}, {@link SameDiff#removeArgumentInterceptor()},
* {@link SameDiff#pauseArgumentInterceptor()}, and {@link SameDiff#unpauseArgumentInterceptor()}
*
*/
public interface ArgumentInterceptor {
SDVariable intercept(SDVariable argument);
}

54
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SDVariable.java
View File

@ -16,6 +16,7 @@
package org.nd4j.autodiff.samediff;
import java.util.Objects;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import onnx.OnnxProto3;
@ -91,7 +92,7 @@ public class SDVariable extends DifferentialFunction implements Serializable {
Preconditions.checkState(dataType != DataType.UNKNOWN, "Unknown datatype is not allowed for SDVariables (variable name: %s)", varName);
String nameScope = sameDiff.currentNameScope();
if(nameScope != null){
if(nameScope != null && !varName.startsWith(nameScope + "/")){
varName = nameScope + "/" + varName;
}
@ -1785,26 +1786,6 @@ public class SDVariable extends DifferentialFunction implements Serializable {
(variableType == VariableType.PLACEHOLDER && shape != null ? ",shape=" + Arrays.toString(shape): "") + ")";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
SDVariable that = (SDVariable) o;
if (varName != null ? !varName.equals(that.varName) : that.varName != null) return false;
return weightInitScheme != null ? weightInitScheme.equals(that.weightInitScheme) : that.weightInitScheme == null;
}
@Override
public int hashCode() {
int result = super.hashCode();
result = 31 * result + (varName != null ? varName.hashCode() : 0);
result = 31 * result + (weightInitScheme != null ? weightInitScheme.hashCode() : 0);
return result;
}
@Override
public String onnxName() {
throw new NoOpNameFoundException("No onnx op opName found for " + opName());
@ -1966,4 +1947,35 @@ public class SDVariable extends DifferentialFunction implements Serializable {
return x;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof SDVariable)) {
return false;
}
SDVariable that = (SDVariable) o;
if (!Objects.equals(varName, that.varName)) {
return false;
}
if (variableType != that.variableType) {
return false;
}
if(sameDiff != that.sameDiff){
return false;
}
return dataType == that.dataType;
}
@Override
public int hashCode() {
int result = super.hashCode();
result = 31 * result + (varName != null ? varName.hashCode() : 0);
result = 31 * result + (variableType != null ? variableType.hashCode() : 0);
result = 31 * result + (dataType != null ? dataType.hashCode() : 0);
return result;
}
}

232
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiff.java
View File

@ -53,6 +53,7 @@ import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
import org.nd4j.linalg.api.ops.impl.controlflow.If;
import org.nd4j.linalg.api.ops.impl.controlflow.While;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Enter;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Merge;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Switch;
import org.nd4j.linalg.api.ops.impl.layers.ExternalErrorsFunction;
import org.nd4j.linalg.api.ops.impl.shape.tensorops.TensorArray;
@ -246,6 +247,14 @@ public class SameDiff extends SDBaseOps {
private boolean resolvedVariables = false;
@Getter
private Stack<ArgumentInterceptor> argumentInterceptors = new Stack<>();
@Getter
private Set<ArgumentInterceptor> pausedArgumentInterceptors = new HashSet<>();
private Set<String> blockNames = new HashSet<>();
@Getter
@Setter
boolean logExecution = true;
@ -472,7 +481,10 @@ public class SameDiff extends SDBaseOps {
if(scope == null){
return name;
}
if(!name.startsWith(scope + "/"))
return scope + "/" + name;
else
return name;
}
//Intentionally package private
@ -533,6 +545,24 @@ public class SameDiff extends SDBaseOps {
}
public List<SameDiffOp> getOpsInScope(NameScope scope){
ArrayList<SameDiffOp> ops = new ArrayList<>();
for(SameDiffOp v : this.ops.values()){
if(v.getName().startsWith(scope.getName()))
ops.add(v);
}
return ops;
}
public List<SDVariable> getVariablesInScope(NameScope scope){
ArrayList<SDVariable> vars = new ArrayList<>();
for(SDVariable v : variables()){
if(v.getVarName().startsWith(scope.getName()))
vars.add(v);
}
return vars;
}
/**
* @param sameDiff
* @return
@ -1109,6 +1139,19 @@ public class SameDiff extends SDBaseOps {
}
}
/**
* Remove a property to resolve added with {@link #addPropertyToResolve(DifferentialFunction, String)}
*
* @param forFunction the function to add the property to resolve for
* @param arrayName the array name
*/
public void removePropertyToResolve(DifferentialFunction forFunction, String arrayName) {
if (propertiesToResolve.containsKey(forFunction.getOwnName())) {
List<String> newVal = propertiesToResolve.get(forFunction.getOwnName());
newVal.remove(arrayName);
}
}
/**
* Return the properties to resolve for the given function.
* This is typically used right before execution in model import in
@ -1272,6 +1315,92 @@ public class SameDiff extends SDBaseOps {
}
}
/**
* Add a new argument interceptor to the interceptor stack
*
* For internal use only.
*
* When a op is added with arguments, most recent argument interceptor is called on it.
* If ops are added in that interceptor, the next most recent will be called on their args, and so on.
*
* @param interceptor the argument interceptor to add
*/
public void addArgumentInterceptor(@NonNull ArgumentInterceptor interceptor){
argumentInterceptors.push(interceptor);
}
private boolean isArgumentInterceptorPaused(@NonNull ArgumentInterceptor interceptor){
return pausedArgumentInterceptors.contains(interceptor);
}
private ArgumentInterceptor getArgumentInterceptorToUse(){
if(argumentInterceptors.isEmpty())
return null;
ArgumentInterceptor use = argumentInterceptors.peek();
int i = 1;
while(isArgumentInterceptorPaused(use)){
if(argumentInterceptors.size() - i < 0)
return null;
use = argumentInterceptors.elementAt(argumentInterceptors.size() - i);
i++;
}
return use;
}
/**
* Remote the top (most recently added) argument interceptor
*
* For internal use only.
*/
public void removeArgumentInterceptor(){
if(!argumentInterceptors.isEmpty())
argumentInterceptors.pop();
}
/**
* Pause the top (most recently added) argument interceptor
*
* For internal use only.
*/
public void pauseArgumentInterceptor(){
pausedArgumentInterceptors.add(argumentInterceptors.peek());
}
/**
* Pause the given argument interceptor
*
* For internal use only.
*
* @param interceptor the argument interceptor to pause
*/
public void pauseArgumentInterceptor(@NonNull ArgumentInterceptor interceptor){
pausedArgumentInterceptors.add(interceptor);
}
/**
* Unpause the top (most recently added) argument interceptor
*
* For internal use only.
*/
public void unpauseArgumentInterceptor(){
pausedArgumentInterceptors.remove(argumentInterceptors.peek());
}
/**
* Unpause the top given argument interceptor
*
* For internal use only.
*
* @param interceptor the argument interceptor to unpause
*/
public void unpauseArgumentInterceptor(@NonNull ArgumentInterceptor interceptor){
pausedArgumentInterceptors.remove(interceptor);
}
/**
* Adds incoming arguments for the specified differential function to the graph
*
@ -1279,6 +1408,17 @@ public class SameDiff extends SDBaseOps {
* @param function Function
*/
public void addArgsFor(String[] variables, DifferentialFunction function) {
ArgumentInterceptor interceptor = getArgumentInterceptorToUse();
if(interceptor != null) {
pauseArgumentInterceptor(interceptor);
for (int i = 0; i < variables.length; i++) {
variables[i] = interceptor.intercept(getVariable(variables[i])).getVarName();
}
unpauseArgumentInterceptor(interceptor);
}
if (function.getOwnName() == null)
throw new ND4JIllegalStateException("Instance id can not be null. Function not initialized properly");
@ -1309,7 +1449,6 @@ public class SameDiff extends SDBaseOps {
}
}
/**
* Adds incoming arguments for the specified differential function to the graph
*
@ -1317,6 +1456,7 @@ public class SameDiff extends SDBaseOps {
* @param function Function
*/
public void addArgsFor(SDVariable[] variables, DifferentialFunction function) {
String[] varNames = new String[variables.length];
for (int i = 0; i < varNames.length; i++) {
if (variables[i] == null)
@ -1326,6 +1466,58 @@ public class SameDiff extends SDBaseOps {
addArgsFor(varNames, function);
}
/**
* Replaces the argument at i with newArg for function
* Does not use (or remove) ArgumentInterceptor stuff
*/
public void replaceArgFor(int i, @NonNull SDVariable newArg, @NonNull DifferentialFunction function){
Preconditions.checkArgument(i < function.args().length, "Index out of range: function " +
function.getOwnName() + " only has " + function.args().length + " args but you are trying" +
"to replace the argument at " + i);
String oldName = function.arg(i).getVarName();
String newName = newArg.getVarName();
if(function.arg(i).isPlaceHolder() && !newArg.isPlaceHolder()){
boolean otherPlaceholders = false;
for(int j = 0 ; j < function.argNames().length ; j++){
if(j == i)
continue;
if(function.arg(j).isPlaceHolder())
otherPlaceholders = true;
}
if(!otherPlaceholders)
placeHolderFunctions.remove(function.getOwnName());
} else if(!function.arg(i).isPlaceHolder() && newArg.isPlaceHolder()){
if(!placeHolderFunctions.contains(function.getOwnName()))
placeHolderFunctions.add(function.getOwnName());
}
List<String> oldArgs = ops.get(function.getOwnName()).getInputsToOp();
oldArgs = new ArrayList<>(oldArgs);
oldArgs.set(i, newName);
ops.get(function.getOwnName()).setInputsToOp(oldArgs);
List<String> funcs = this.variables.get(newName).getInputsForOp();
if (funcs == null) {
funcs = new ArrayList<>();
this.variables.get(newName).setInputsForOp(funcs);
}
if(!funcs.contains(function.getOwnName())) //Avoid duplicates for function names.
funcs.add(function.getOwnName());
List<String> oldFuncs = this.variables.get(oldName).getInputsForOp();
if(oldFuncs != null) {
if(!ArrayUtils.contains(function.argNames(), oldName))
oldFuncs.remove(function.getOwnName());
}
}
/**
* Get the differential function (if any) that this variable is the output for
*
@ -1519,6 +1711,7 @@ public class SameDiff extends SDBaseOps {
//A bit of a hack for TF import: some TF graphs have Switch ops, where the output of one branch isn't consumed
// by any ops. Consequently, during execution this "output" might never be available. So we'll exclude the output of execution here
// This applies to SameDiff while loops as well
if(o.getOp() instanceof Switch){
continue;
}
@ -2239,6 +2432,7 @@ public class SameDiff extends SDBaseOps {
if (name == null || name.length() < 1)
name = getNewVarName();
SDVariable v = new SDVariable(name, VariableType.CONSTANT, this, constant.shape(), constant.dataType(), null);
name = v.getVarName();
variables.put(name, Variable.builder().name(name).variable(v).build());
constantArrays.put(name, new DeviceLocalNDArray(constant));
return v;
@ -2305,6 +2499,7 @@ public class SameDiff extends SDBaseOps {
public SDVariable var(@NonNull String name, @NonNull VariableType variableType, WeightInitScheme weightInitScheme,
org.nd4j.linalg.api.buffer.DataType dataType, long... shape) {
String withScope = nameWithScope(name);
if (variables.containsKey(withScope)) {
if(nameScopes.isEmpty()){
throw new IllegalArgumentException("Another variable with the name " + name + " already exists (current name scope: \""
@ -3414,12 +3609,9 @@ public class SameDiff extends SDBaseOps {
/**
* Creates a while statement
*
* @param sameDiffConditional
* @param loopBody
* @return
* @deprecated Use {@link SDBaseOps#whileLoop(String[], String, SDVariable[], SameDiffSingleLambda, SameDiffLambda)}
*/
@Deprecated
public While whileStatement(SameDiffConditional sameDiffConditional,
SameDiffFunctionDefinition conditionBody,
SameDiffFunctionDefinition loopBody
@ -3435,11 +3627,9 @@ public class SameDiff extends SDBaseOps {
}
/**
* @param conditional
* @param trueBody
* @param falseBody
* @return
* @deprecated Use {@link SDBaseOps#ifCond(String, String, SameDiffNoArgSingleLambda, SameDiffNoArgSingleLambda, SameDiffNoArgSingleLambda)}
*/
@Deprecated
public If ifStatement(SameDiffConditional conditional,
SameDiffFunctionDefinition conditionBody,
SameDiffFunctionDefinition trueBody,
@ -5466,5 +5656,27 @@ public class SameDiff extends SDBaseOps {
return out;
}
/**
* For internal use only.
* Creates a new discinct block name from baseName.
* Block names are used by If and While
*/
public String newBlockName(String baseName){
if(baseName == null)
return null;
if(!blockNames.contains(baseName)){
blockNames.add(baseName);
return baseName;
} else {
int i = 1;
while(blockNames.contains(baseName + "_" + i)){
i++;
}
blockNames.add(baseName + "_" + i);
return baseName + "_" + i;
}
}
}

24
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiffLambda.java Normal file
View File

@ -0,0 +1,24 @@
/*******************************************************************************
* Copyright (c) 2015-2019 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.nd4j.autodiff.samediff;
/**
* A basic SameDiff lambda, used in while loop creation (the body).
*/
public interface SameDiffLambda {
SDVariable[] define(SameDiff sameDiff, SDVariable[] inputs);
}

24
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiffNoArgSingleLambda.java Normal file
View File

@ -0,0 +1,24 @@
/*******************************************************************************
* Copyright (c) 2015-2019 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.nd4j.autodiff.samediff;
/**
* A SameDiff lambda with only one output and no arguments. Used in if condition creation (the condition and bodies).
*/
public interface SameDiffNoArgSingleLambda {
SDVariable define(SameDiff sameDiff);
}

24
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiffSingleLambda.java Normal file
View File

@ -0,0 +1,24 @@
/*******************************************************************************
* Copyright (c) 2015-2019 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.nd4j.autodiff.samediff;
/**
* A SameDiff lambda with only one output, used in while loop creation (the condition).
*/
public interface SameDiffSingleLambda {
SDVariable define(SameDiff sameDiff, SDVariable[] inputs);
}

313
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java
View File

@ -16,12 +16,25 @@
package org.nd4j.autodiff.samediff.ops;
import com.google.common.collect.Sets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import lombok.NonNull;
import org.nd4j.autodiff.functions.DifferentialFunctionFactory;
import org.nd4j.autodiff.samediff.ArgumentInterceptor;
import org.nd4j.autodiff.samediff.NameScope;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.autodiff.samediff.SameDiffFunctionDefinition;
import org.nd4j.autodiff.samediff.SameDiffLambda;
import org.nd4j.autodiff.samediff.SameDiffNoArgSingleLambda;
import org.nd4j.autodiff.samediff.SameDiffSingleLambda;
import org.nd4j.autodiff.samediff.internal.SameDiffOp;
import org.nd4j.linalg.api.blas.params.MMulTranspose;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Merge;
import org.nd4j.linalg.api.ops.impl.shape.OneHot;
import org.nd4j.linalg.api.ops.impl.transforms.gradient.GradientBackwardsMarker;
import org.nd4j.linalg.indexing.conditions.Condition;
@ -3142,4 +3155,304 @@ public abstract class SDBaseOps {
SDVariable ret = f().zerosLike(name, input);
return updateVariableNameAndReference(ret, name);
}
/**
* See {@link #any(String, SDVariable, int...)}
*/
public SDVariable any(SDVariable x, int... dimensions){
return any(null, x, dimensions);
}
//TODO check any w/ no dimensions
/**
* Boolean or array reduction operation, optionally along specified dimensions
*
* @param name Name of the output variable
* @param x Input variable
* @param dimensions Dimensions to reduce over. If dimensions are not specified, full array reduction is performed
* @return Output variable: reduced array of rank (input rank - num dimensions)
*/
public SDVariable any(String name, SDVariable x, int... dimensions){
validateBool("any", x);
SDVariable ret = f().any(x, dimensions);
return updateVariableNameAndReference(ret, name);
}
/**
* See {@link #all(String, SDVariable, int...)}
*/
public SDVariable all(SDVariable x, int... dimensions){
return all(null, x, dimensions);
}
/**
* Boolean and array reduction operation, optionally along specified dimensions
*
* @param name Name of the output variable
* @param x Input variable
* @param dimensions Dimensions to reduce over. If dimensions are not specified, full array reduction is performed
* @return Output variable: reduced array of rank (input rank - num dimensions)
*/
public SDVariable all(String name, SDVariable x, int... dimensions){
validateBool("all", x);
SDVariable ret = f().all(x, dimensions);
return updateVariableNameAndReference(ret, name);
}
/**
* See {@link #whileLoop(String[], String, SDVariable[], SameDiffSingleLambda, SameDiffLambda)}
*/
public SDVariable[] whileLoop(@NonNull SDVariable[] loopVars,
@NonNull SameDiffSingleLambda cond, @NonNull SameDiffLambda body){
return whileLoop(null, null, loopVars, cond, body);
}
/**
* See {@link #whileLoop(String[], String, SDVariable[], SameDiffSingleLambda, SameDiffLambda)}
*/
public SDVariable[] whileLoop(String loopName, @NonNull SDVariable[] loopVars,
@NonNull SameDiffSingleLambda cond, @NonNull SameDiffLambda body){
return whileLoop(null, loopName, loopVars, cond, body);
}
/**
* Constructs a While loop using the tensorflow style control flow operations (Switch, Merge, Enter, Exit, and NextIteration)
*
* Repeatedly executes body on the loop variables and updates them with the results, until cond evaluates to false
*
* Note that cond and body lambdas are only called once to construct the graph. The constructed graph is used for further iterations.
*
* See <a href="http://download.tensorflow.org/paper/white_paper_tf_control_flow_implementation_2017_11_1.pdf">Tensorflow Control Flow Implementation</a>
*
* @param outputNames Names to give the output variables. If null, doesn't rename
* @param loopName The name of the loop block and frame (must be unique). If null, uses "if"
* @param loopVars Loop variables' inputs
* @param cond A lambda evaluating to the loop condition
* @param body A lambda doing the loop operation and returning the new loop variable values
* @return The values of the loop variables once condition is false
*/
public SDVariable[] whileLoop(String[] outputNames, final String loopName, @NonNull SDVariable[] loopVars,
@NonNull SameDiffSingleLambda cond, @NonNull SameDiffLambda body){
final String frameName = sd().newBlockName(loopName == null ? "while" : loopName);
NameScope loopScope = sd().withNameScope(frameName);
//SDVariable counter = SD.scalar(SD.generateNewVarName("counter", 0), 0);
SDVariable[] entered = new SDVariable[loopVars.length];
for(int i = 0 ; i < loopVars.length ; i++){
entered[i] = f().enter(loopVars[i], frameName);
}
//counter = SD.f().enter(counter, frameName);
SDVariable[] merged = new SDVariable[loopVars.length];
Merge[] mergeOps = new Merge[loopVars.length];
for(int i = 0 ; i < loopVars.length ; i++){
// the second arg will later be replaced with the output of NextIteration
// but that isn't available yet (and can't be, as it depends on this)
mergeOps[i] = new Merge(sd(), entered[i], entered[i]);
merged[i] = mergeOps[i].outputVariable();
}
//Merge counterMerge = new Merge(SD, counter, counter);
//counter = counterMerge.outputVariable();
NameScope condScope = sd().withNameScope("cond");
SDVariable cond_result = cond.define(sd(), merged);
condScope.close();
if (cond_result.dataType() != DataType.BOOL)
throw new IllegalStateException("Can not use " + cond_result.getVarName() + " as the condition of an While loop, the condition must be a boolean.");
final Set<String> alreadyEntered = Sets.newHashSet();
SDVariable[] trueSwitches = new SDVariable[loopVars.length];
SDVariable[] exits = new SDVariable[loopVars.length];
for(int i = 0 ; i < loopVars.length ; i++){
SDVariable[] s = f().switchOp(merged[i], cond_result);
trueSwitches[i] = s[1];
alreadyEntered.add(s[1].getVarName());
exits[i] = f().exit(s[0]);
}
//SDVariable[] cs = SD.f().switchOp(counter, cond_result);
//SDVariable counterExit = SD.f().exit(cs[0]);
//counter = cs[1];
final Set<String> declared = Sets.newHashSet(sd().variableMap().keySet());
final Map<String, SDVariable> done = new HashMap<>();
sd().addArgumentInterceptor(new ArgumentInterceptor() {
@Override
public SDVariable intercept(SDVariable argument) {
if(!declared.contains(argument.getVarName()))
return argument;
if(alreadyEntered.contains(argument.getVarName()))
return argument;
if(done.containsKey(argument.getVarName()))
return done.get(argument.getVarName());
SDVariable e = f().enter(argument, frameName, true);
done.put(argument.getVarName(), e);
return e;
}
});
NameScope bodyScope = sd().withNameScope("body");
SDVariable[] outs = body.define(sd(), trueSwitches);
bodyScope.close();
sd().removeArgumentInterceptor();
//counter.add(1);
for(int i = 0 ; i < loopVars.length ; i++){
SDVariable n = f().nextIteration(outs[i]);
mergeOps[i].replaceArg(1,n);
}
//counterMerge.replaceArg(1, counter);
loopScope.close();
return updateVariableNamesAndReferences(exits, outputNames);
}
/**
* See {@link #ifCond(String, String, SameDiffNoArgSingleLambda, SameDiffNoArgSingleLambda, SameDiffNoArgSingleLambda)}
*/
public SDVariable ifCond(@NonNull SameDiffNoArgSingleLambda cond,
@NonNull SameDiffNoArgSingleLambda trueBody, @NonNull SameDiffNoArgSingleLambda falseBody){
return ifCond(null, null, cond, trueBody, falseBody);
}
/**
* See {@link #ifCond(String, String, SameDiffNoArgSingleLambda, SameDiffNoArgSingleLambda, SameDiffNoArgSingleLambda)}
*/
public SDVariable ifCond(String ifName, @NonNull SameDiffNoArgSingleLambda cond,
@NonNull SameDiffNoArgSingleLambda trueBody, @NonNull SameDiffNoArgSingleLambda falseBody){
return ifCond(null, ifName, cond, trueBody, falseBody);
}
/**
* Constructs a If statement using the tensorflow style control flow operations (Switch and Merge)
*
* If the result of cond is true, returns the result of trueBody, otherwise returns the result of falseBody
*
* Note that cond and body lambdas are only called once to construct the graph. The constructed graph is used to evaluate.
*
* See <a href="http://download.tensorflow.org/paper/white_paper_tf_control_flow_implementation_2017_11_1.pdf">Tensorflow Control Flow Implementation</a>
*
* @param outputName Name to give the output variable. If null, doesn't rename
* @param ifName The name of the if block. If null, uses "if"
* @param cond A lambda evaluating to the if condition
* @param trueBody A lambda to be executed if cond is true (the if block)
* @param falseBody A lambda to be executed if cond is false (the else block)
* @return The value of trueBody if cond is true, or falseBody if it isn't
*/
public SDVariable ifCond(String outputName, String ifName, @NonNull SameDiffNoArgSingleLambda cond,
@NonNull SameDiffNoArgSingleLambda trueBody, @NonNull SameDiffNoArgSingleLambda falseBody){
ifName = sd().newBlockName(ifName == null ? "if" : ifName);
NameScope ifScope = sd().withNameScope(ifName);
NameScope condScope = sd().withNameScope("cond");
final SDVariable pred = cond.define(sd());
condScope.close();
if (pred.dataType() != DataType.BOOL) {
//cleanup partially added block
for(SDVariable v : sd().getVariablesInScope(ifScope))
sd().getVariables().remove(v.getVarName());
for(SameDiffOp op : sd().getOpsInScope(ifScope)) {
for(String in : op.getInputsToOp()){
sd().removeArgFromFunction(in, op.getOp());
}
sd().getOps().remove(op.getName());
}
throw new IllegalStateException("Can not use " + pred.getVarName()
+ " as the condition of an If statement, the condition must be a boolean.");
}
final Map<String, SDVariable[]> switches = new HashMap<>();
final Set<String> declared = Sets.newHashSet(sd().variableMap().keySet());
sd().addArgumentInterceptor(new ArgumentInterceptor() {
@Override
public SDVariable intercept(SDVariable argument) {
// if its declared in the if, we don't care acout it
if(!declared.contains(argument.getVarName()))
return argument;
// if we've already added a switch, move on
if(switches.containsKey(argument.getVarName()))
return switches.get(argument.getVarName())[1];
SDVariable[] s = f().switchOp(argument, pred);
switches.put(argument.getVarName(), s);
return s[1];
}
});
NameScope trueScope = sd().withNameScope("trueBody");
SDVariable trueOut = trueBody.define(sd());
sd().removeArgumentInterceptor();
if(declared.contains(trueOut.getVarName())) {
SDVariable[] s = f().switchOp(trueOut, pred);
switches.put(trueOut.getVarName(), s);
trueOut = s[1];
}
trueScope.close();
final Set<String> declared2 = Sets.newHashSet(sd().variableMap().keySet());
sd().addArgumentInterceptor(new ArgumentInterceptor() {
@Override
public SDVariable intercept(SDVariable argument) {
// if its declared in the if, we don't care acout it
if(!declared2.contains(argument.getVarName()))
return argument;
// if we've already added a switch, move on
if(switches.containsKey(argument.getVarName()))
return switches.get(argument.getVarName())[0];
SDVariable[] s = f().switchOp(argument, pred);
switches.put(argument.getVarName(), s);
return s[0];
}
});
NameScope falseScope = sd().withNameScope("falseBody");
SDVariable falseOut = falseBody.define(sd());
sd().removeArgumentInterceptor();
if(declared2.contains(falseOut.getVarName())) {
SDVariable[] s = f().switchOp(falseOut, pred);
switches.put(falseOut.getVarName(), s);
falseOut = s[0];
}
falseScope.close();
SDVariable output = f().merge(trueOut, falseOut);
ifScope.close();
return updateVariableNameAndReference(output, outputName);
}
}

45
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java
View File

@ -411,6 +411,29 @@ public class SDNN extends SDOps {
return updateVariableNameAndReference(ret, name);
}
/**
* Log softmax activation
*
* @param x Input variable
* @return Output variable
*/
public SDVariable logSoftmax(SDVariable x, int dimension) {
return logSoftmax(null, x, dimension);
}
/**
* Log softmax activation
*
* @param name Variable name
* @param x Input variable
* @return Output variable
*/
public SDVariable logSoftmax(String name, SDVariable x, int dimension) {
validateFloatingPoint("log softmax", x);
SDVariable ret = f().logSoftmax(x, dimension);
return updateVariableNameAndReference(ret, name);
}
/**
* Element-wise rectified linear function with specified cutoff:<br>
* out[i] = in[i] if in[i] >= cutoff
@ -591,6 +614,28 @@ public class SDNN extends SDOps {
return updateVariableNameAndReference(result, name);
}
/**
* Softmax activation
*
* @param x Input variable
* @return Output variable
*/
public SDVariable softmax(SDVariable x, int dimension) {
return softmax(null, x, dimension);
}
/**
* Softmax activation
*
* @param x Input variable
* @return Output variable
*/
public SDVariable softmax(String name, SDVariable x, int dimension) {
validateFloatingPoint("softmax", x);
SDVariable result = f().softmax(x, dimension);
return updateVariableNameAndReference(result, name);
}
/**
* @param x
* @return

362
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java
View File

@ -17,36 +17,47 @@
package org.nd4j.autodiff.samediff.serde;
import com.google.flatbuffers.FlatBufferBuilder;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import lombok.NonNull;
import lombok.val;
import org.nd4j.autodiff.functions.DifferentialFunction;
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.autodiff.samediff.VariableType;
import org.nd4j.base.Preconditions;
import org.nd4j.graph.*;
import org.nd4j.graph.DataType;
import org.nd4j.graph.FlatArray;
import org.nd4j.graph.FlatNode;
import org.nd4j.graph.FlatProperties;
import org.nd4j.graph.IntPair;
import org.nd4j.graph.OpType;
import org.nd4j.graph.VarType;
import org.nd4j.imports.converters.DifferentialFunctionClassHolder;
import org.nd4j.linalg.api.buffer.DataBuffer;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.*;
import org.nd4j.linalg.api.ops.BaseIndexAccumulation;
import org.nd4j.linalg.api.ops.BaseReduceOp;
import org.nd4j.linalg.api.ops.CustomOp;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.Op.Type;
import org.nd4j.linalg.api.ops.ScalarOp;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Enter;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Exit;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Merge;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.NextIteration;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Switch;
import org.nd4j.linalg.api.shape.Shape;
import org.nd4j.linalg.exception.ND4JIllegalStateException;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.primitives.Pair;
import org.nd4j.linalg.util.ArrayUtil;
import java.nio.ByteOrder;
import java.util.*;
public class FlatBuffersMapper {
private FlatBuffersMapper(){ }
private FlatBuffersMapper() {
}
/**
* This method converts enums for DataType
*
* @param type
* @return
*/
public static byte getDataTypeAsByte(@NonNull org.nd4j.linalg.api.buffer.DataType type) {
switch (type) {
@ -84,80 +95,79 @@ public class FlatBuffersMapper {
/**
* This method converts enums for DataType
*
* @param val
* @return
*/
public static org.nd4j.linalg.api.buffer.DataType getDataTypeFromByte(byte val) {
if (val == DataType.FLOAT)
if (val == DataType.FLOAT) {
return org.nd4j.linalg.api.buffer.DataType.FLOAT;
else if (val == DataType.DOUBLE)
} else if (val == DataType.DOUBLE) {
return org.nd4j.linalg.api.buffer.DataType.DOUBLE;
else if (val == DataType.HALF)
} else if (val == DataType.HALF) {
return org.nd4j.linalg.api.buffer.DataType.HALF;
else if (val == DataType.INT32)
} else if (val == DataType.INT32) {
return org.nd4j.linalg.api.buffer.DataType.INT;
else if (val == DataType.INT64)
} else if (val == DataType.INT64) {
return org.nd4j.linalg.api.buffer.DataType.LONG;
else if (val == DataType.INT8)
} else if (val == DataType.INT8) {
return org.nd4j.linalg.api.buffer.DataType.BYTE;
else if (val == DataType.BOOL)
} else if (val == DataType.BOOL) {
return org.nd4j.linalg.api.buffer.DataType.BOOL;
else if (val == DataType.UINT8)
} else if (val == DataType.UINT8) {
return org.nd4j.linalg.api.buffer.DataType.UBYTE;
else if (val == DataType.INT16)
} else if (val == DataType.INT16) {
return org.nd4j.linalg.api.buffer.DataType.SHORT;
else if (val == DataType.UTF8)
} else if (val == DataType.UTF8) {
return org.nd4j.linalg.api.buffer.DataType.UTF8;
else if (val == DataType.UINT16)
} else if (val == DataType.UINT16) {
return org.nd4j.linalg.api.buffer.DataType.UINT16;
else if (val == DataType.UINT32)
} else if (val == DataType.UINT32) {
return org.nd4j.linalg.api.buffer.DataType.UINT32;
else if (val == DataType.UINT64)
} else if (val == DataType.UINT64) {
return org.nd4j.linalg.api.buffer.DataType.UINT64;
else
} else {
throw new RuntimeException("Unknown datatype: " + val);
}
}
/**
* This method return operation ID for given op name/type pair.
*
* @param name
* @param type
* @return
*/
public static long getOpNum(String name, Op.Type type) {
if (type == Op.Type.LOOP) {
return 0;
} else if (type == Op.Type.RETURN) {
return 40;
} else if (type == Op.Type.IF) {
return 30;
} else if (type == Op.Type.CONDITIONAL) {
return 10;
} else if (type == Op.Type.MERGE) {
return 60L;
} else if (type == Op.Type.LOOP_COND) {
return 70L;
} else if (type == Op.Type.NEXT_ITERATION) {
return 80L;
} else if (type == Op.Type.EXIT) {
return 90L;
} else if (type == Op.Type.ENTER) {
return 100L;
} else if (type == Type.LOGIC) {
switch (name) {
case Enter.OP_NAME:
return Enter.OP_NUM;
case Exit.OP_NAME:
return Exit.OP_NUM;
case NextIteration.OP_NAME:
return NextIteration.OP_NUM;
case Merge.OP_NAME:
return Merge.OP_NUM;
case Switch.OP_NAME:
return Switch.OP_NUM;
default:
throw new IllegalStateException("Unknown LOGIC op with name: " + name);
}
} else if (type == Op.Type.CUSTOM) {
val name2 = Nd4j.getExecutioner().getCustomOperations().get(name.toLowerCase());
if (name2 == null) {
val name3 = Nd4j.getExecutioner().getCustomOperations().get(name);
if (name3 == null)
if (name3 == null) {
return 0;
else
} else {
return name3.getHash();
} else
}
} else {
return name2.getHash();
}
//return Nd4j.getExecutioner().getCustomOperations().get(name.toLowerCase()).getHash();
} else {
@ -165,7 +175,7 @@ public class FlatBuffersMapper {
DifferentialFunction op = DifferentialFunctionClassHolder.getInstance().getInstance(name);
return op.opNum();
} catch (Exception e) {
throw new RuntimeException("Could not find op number for operation: [" + name + "]",e);
throw new RuntimeException("Could not find op number for operation: [" + name + "]", e);
}
}
}
@ -212,7 +222,7 @@ public class FlatBuffersMapper {
case OpType.RANDOM:
return Op.Type.RANDOM;
case OpType.LOGIC:
return Op.Type.META;
return Type.LOGIC;
case OpType.CUSTOM:
return Op.Type.CUSTOM;
case OpType.PAIRWISE:
@ -269,15 +279,11 @@ public class FlatBuffersMapper {
return OpType.INDEX_REDUCE;
case RANDOM:
return OpType.RANDOM;
case MERGE:
case CONDITIONAL:
case LOOP:
case RETURN:
case ENTER:
case EXIT:
case NEXT_ITERATION:
case LOOP_COND:
case IF:
case LOGIC:
return OpType.LOGIC;
case CUSTOM:
return OpType.CUSTOM;
@ -295,88 +301,87 @@ public class FlatBuffersMapper {
/**
* This method just converts enums
*
* @param val
* @return
*/
public static ByteOrder getOrderFromByte(byte val) {
if (val == org.nd4j.graph.ByteOrder.LE)
if (val == org.nd4j.graph.ByteOrder.LE) {
return ByteOrder.LITTLE_ENDIAN;
else
} else {
return ByteOrder.BIG_ENDIAN;
}
}
/**
* This method returns current byte order for this JVM as libnd4j enum
*
* @return
*/
public static byte getOrderAsByte() {
if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN))
if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
return org.nd4j.graph.ByteOrder.BE;
else
} else {
return org.nd4j.graph.ByteOrder.LE;
}
}
public static DifferentialFunction fromFlatNode(FlatNode fn){
public static DifferentialFunction fromFlatNode(FlatNode fn) {
int id = fn.id(); //ID of the node
String name = fn.name(); //Name of the node, NOT the name of the op
Op.Type opType = FlatBuffersMapper.getTypeFromByte(fn.opType());
long opNum = fn.opNum(); //Op num: hash for custom, number for legacy
int[] input = new int[fn.inputLength()];
for( int i=0; i<input.length; i++ ){
for (int i = 0; i < input.length; i++) {
input[i] = fn.input(i);
}
IntPair[] inputPaired = new IntPair[fn.inputPairedLength()];
for( int i=0; i<inputPaired.length; i++ ){
for (int i = 0; i < inputPaired.length; i++) {
inputPaired[i] = fn.inputPaired(i);
}
int[] output = new int[fn.outputLength()];
for( int i=0; i<output.length; i++ ){
for (int i = 0; i < output.length; i++) {
output[i] = fn.output(i);
}
double[] extraParams = new double[fn.extraParamsLength()];
for( int i=0; i<extraParams.length; i++ ){
for (int i = 0; i < extraParams.length; i++) {
extraParams[i] = fn.extraParams(i);
}
long[] extraInteger = new long[fn.extraIntegerLength()];
for( int i=0; i<extraInteger.length; i++ ){
for (int i = 0; i < extraInteger.length; i++) {
extraInteger[i] = fn.extraInteger(i);
}
boolean[] extraBools = new boolean[fn.extraBoolsLength()];
for( int i=0; i<extraBools.length; i++ ){
for (int i = 0; i < extraBools.length; i++) {
extraBools[i] = fn.extraBools(i);
}
int[] dimensions = new int[fn.dimensionsLength()];
for( int i=0; i<dimensions.length; i++ ){
for (int i = 0; i < dimensions.length; i++) {
dimensions[i] = fn.dimensions(i);
}
FlatArray fa = fn.scalar();
INDArray scalar = null;
if(fa != null){
if (fa != null) {
scalar = Nd4j.createFromFlatArray(fa);
}
FlatProperties[] flatProperties = new FlatProperties[fn.propertiesLength()];
for( int i=0; i<flatProperties.length; i++ ){
for (int i = 0; i < flatProperties.length; i++) {
flatProperties[i] = fn.properties(i);
}
Map<String,Object> props = FlatBuffersMapper.mapFlatPropertiesToFunctionProperties(Arrays.asList(flatProperties));
Map<String, Object> props = FlatBuffersMapper
.mapFlatPropertiesToFunctionProperties(Arrays.asList(flatProperties));
if(opType == Op.Type.CUSTOM) {
if (opType == Op.Type.CUSTOM || opType == Type.LOGIC) {
String opName = fn.opName();
DifferentialFunction op;
Class<?> c = DifferentialFunctionClassHolder.getInstance().customOpClassForHashAndName(opNum, opName);
Preconditions.checkNotNull(c, "Could not find class for hash %s", opNum);
DifferentialFunction op;
try {
op = (DifferentialFunction) c.newInstance();
} catch (IllegalAccessException | InstantiationException e) {
throw new RuntimeException("Error creating differential function instance of type " + c);
}
op.setOwnName(name);
//Set input SDVariables:
@ -390,7 +395,7 @@ public class FlatBuffersMapper {
op.setPropertiesForFunction(props);
return op;
} else {
Class<?> c = LegacyOpMapper.getLegacyOpClassForId(opType, (int)opNum);
Class<?> c = LegacyOpMapper.getLegacyOpClassForId(opType, (int) opNum);
Op op;
try {
op = (Op) c.newInstance();
@ -398,7 +403,7 @@ public class FlatBuffersMapper {
throw new RuntimeException("Error creating differential function (Op) instance of type " + c);
}
if(extraParams.length > 0) {
if (extraParams.length > 0) {
//Assume that extraParams length 0 means extraArgs was originally null, NOT originally length 0
Object[] extraParamsObj = new Object[extraParams.length];
for (int i = 0; i < extraParams.length; i++) {
@ -406,16 +411,18 @@ public class FlatBuffersMapper {
}
op.setExtraArgs(extraParamsObj);
}
if(opType == Op.Type.SCALAR || opType == Op.Type.SCALAR_BOOL){
ScalarOp sOp = (ScalarOp)op;
if (opType == Op.Type.SCALAR || opType == Op.Type.SCALAR_BOOL) {
ScalarOp sOp = (ScalarOp) op;
sOp.setScalar(scalar);
} else if(opType == Op.Type.REDUCE_FLOAT || opType == Op.Type.REDUCE3 || opType == Op.Type.SUMMARYSTATS || opType == Op.Type.VARIANCE
|| opType == Op.Type.REDUCE_BOOL || opType == Op.Type.REDUCE_LONG || opType == Op.Type.REDUCE_SAME) {
} else if (opType == Op.Type.REDUCE_FLOAT || opType == Op.Type.REDUCE3 || opType == Op.Type.SUMMARYSTATS
|| opType == Op.Type.VARIANCE
|| opType == Op.Type.REDUCE_BOOL || opType == Op.Type.REDUCE_LONG
|| opType == Op.Type.REDUCE_SAME) {
val ba = (BaseReduceOp) op; //Reduce3 ops are also all BaseAccumulations
ba.setDimensions(dimensions);
ba.setDimensionz(Shape.ndArrayDimFromInt(dimensions));
} else if(opType == Op.Type.INDEXREDUCE){
BaseIndexAccumulation bia = (BaseIndexAccumulation)op;
} else if (opType == Op.Type.INDEXREDUCE) {
BaseIndexAccumulation bia = (BaseIndexAccumulation) op;
bia.setDimensions(dimensions);
bia.setDimensionz(Shape.ndArrayDimFromInt(dimensions));
}
@ -428,8 +435,8 @@ public class FlatBuffersMapper {
TRANSFORM_SAME - Abs, Ceil, etc
*/
((DifferentialFunction)op).setPropertiesForFunction(props);
return (DifferentialFunction)op;
((DifferentialFunction) op).setPropertiesForFunction(props);
return (DifferentialFunction) op;
}
}
@ -438,11 +445,11 @@ public class FlatBuffersMapper {
private static final long[] EMPTY_LONG = new long[0];
private static final double[] EMPTY_DOUBLE = new double[0];
public static int[] mapFunctionPropertiesToFlatProperties(FlatBufferBuilder fbb, Map<String,Object> fnProps){
public static int[] mapFunctionPropertiesToFlatProperties(FlatBufferBuilder fbb, Map<String, Object> fnProps) {
int[] outIdxs = new int[fnProps.size()];
int count = 0;
for(Map.Entry<String,Object> e : fnProps.entrySet()){
for (Map.Entry<String, Object> e : fnProps.entrySet()) {
//Possible types here: primitives (as Number objects), primitive arrays, Strings, String arrays, multi-dimensional string/primitives
Object v = e.getValue();
int iname = fbb.createString(e.getKey());
@ -455,13 +462,11 @@ public class FlatBuffersMapper {
int[] sIdx = null;
int[] shape = null;
if(v == null) {
if (v == null) {
//No op
} else if(v instanceof Boolean){
b = new boolean[]{(Boolean)v};
} else if(v instanceof Number) {
} else if (v instanceof Boolean) {
b = new boolean[]{(Boolean) v};
} else if (v instanceof Number) {
if (v instanceof Double) {
d = new double[]{(Double) v};
} else if (v instanceof Integer) {
@ -469,39 +474,41 @@ public class FlatBuffersMapper {
} else if (v instanceof Long) {
l = new long[]{(Long) v};
} else {
throw new UnsupportedOperationException("Unable to map property \"" + e.getKey() + "\" of type " + v.getClass());
throw new UnsupportedOperationException(
"Unable to map property \"" + e.getKey() + "\" of type " + v.getClass());
}
} else if(v instanceof String) {
} else if (v instanceof String) {
String str = (String) v;
int strOffset = fbb.createString(str);
sIdx = new int[]{strOffset};
} else if(v instanceof org.nd4j.linalg.api.buffer.DataType ) {
} else if (v instanceof org.nd4j.linalg.api.buffer.DataType) {
String str = v.toString();
int strOffset = fbb.createString(str);
sIdx = new int[]{strOffset};
} else if(v instanceof Enum){
} else if (v instanceof Enum) {
String str = v.toString();
int strOffset = fbb.createString(str);
sIdx = new int[]{strOffset};
} else if(v instanceof INDArray){
INDArray arr = (INDArray)v;
} else if (v instanceof INDArray) {
INDArray arr = (INDArray) v;
aIdx = new int[]{arr.toFlatArray(fbb)};
} else if(v.getClass().isArray()){
if(v.getClass().getComponentType().isPrimitive()){
if(v instanceof boolean[]) {
b = (boolean[])v;
} else if (v.getClass().isArray()) {
if (v.getClass().getComponentType().isPrimitive()) {
if (v instanceof boolean[]) {
b = (boolean[]) v;
shape = new int[]{b.length};
} else if(v instanceof double[]){
d = (double[])v;
} else if (v instanceof double[]) {
d = (double[]) v;
shape = new int[]{d.length};
} else if(v instanceof int[]){
i = (int[])v;
} else if (v instanceof int[]) {
i = (int[]) v;
shape = new int[]{i.length};
} else if(v instanceof long[]){
l = (long[])v;
} else if (v instanceof long[]) {
l = (long[]) v;
shape = new int[]{l.length};
} else {
throw new UnsupportedOperationException("Unable to map property \"" + e.getKey() + "\" of type " + v.getClass());
throw new UnsupportedOperationException(
"Unable to map property \"" + e.getKey() + "\" of type " + v.getClass());
}
} else if (v instanceof String[]) {
//String[]
@ -511,33 +518,35 @@ public class FlatBuffersMapper {
sIdx[j] = fbb.createString(strArr[j]);
}
shape = new int[]{strArr.length};
} else if (v instanceof INDArray[]){
INDArray[] arrArr = (INDArray[])v;
} else if (v instanceof INDArray[]) {
INDArray[] arrArr = (INDArray[]) v;
aIdx = new int[arrArr.length];
for( int j=0; j<arrArr.length; j++){
for (int j = 0; j < arrArr.length; j++) {
aIdx[j] = arrArr[j].toFlatArray(fbb);
}
} else if(v.getClass().getComponentType().isArray()){
} else if (v.getClass().getComponentType().isArray()) {
shape = ArrayUtil.arrayShape(v, true);
//Multi-dimensional array
if(v instanceof boolean[][]) {
if (v instanceof boolean[][]) {
b = ArrayUtil.flatten((boolean[][]) v);
} else if(v instanceof boolean[][][]){
} else if (v instanceof boolean[][][]) {
b = ArrayUtil.flatten((boolean[][][]) v);
} else if(v instanceof double[][]){
} else if (v instanceof double[][]) {
d = ArrayUtil.flatten((double[][]) v);
} else if(v instanceof double[][][]){
} else if (v instanceof double[][][]) {
d = ArrayUtil.flatten((double[][][]) v);
} else if(v instanceof int[][]){
i = ArrayUtil.flatten((int[][])v);
} else if(v instanceof int[][][]){
i = ArrayUtil.flatten((int[][][])v);
} else if(v instanceof long[][]){
l = ArrayUtil.flatten((long[][])v);
} else if(v instanceof long[][][]){
l = ArrayUtil.flatten((long[][][])v);
} else if (v instanceof int[][]) {
i = ArrayUtil.flatten((int[][]) v);
} else if (v instanceof int[][][]) {
i = ArrayUtil.flatten((int[][][]) v);
} else if (v instanceof long[][]) {
l = ArrayUtil.flatten((long[][]) v);
} else if (v instanceof long[][][]) {
l = ArrayUtil.flatten((long[][][]) v);
} else {
throw new UnsupportedOperationException("Unable to map multidimensional array property \"" + e.getKey() + "\" of type " + v.getClass());
throw new UnsupportedOperationException(
"Unable to map multidimensional array property \"" + e.getKey() + "\" of type " + v
.getClass());
}
}
}
@ -550,21 +559,22 @@ public class FlatBuffersMapper {
int idxS = FlatProperties.createSVector(fbb, sIdx != null ? sIdx : EMPTY_INT);
int idxShape = FlatProperties.createShapeVector(fbb, shape != null ? shape : EMPTY_INT);
outIdxs[count++] = FlatProperties.createFlatProperties(fbb, iname, idxI, idxL, idxD, idxA, idxB, idxS, idxShape);
outIdxs[count++] = FlatProperties
.createFlatProperties(fbb, iname, idxI, idxL, idxD, idxA, idxB, idxS, idxShape);
}
return outIdxs;
}
public static Map<String,Object> mapFlatPropertiesToFunctionProperties(Iterable<FlatProperties> list){
Map<String,Object> out = new HashMap<>();
for(FlatProperties p : list){
public static Map<String, Object> mapFlatPropertiesToFunctionProperties(Iterable<FlatProperties> list) {
Map<String, Object> out = new HashMap<>();
for (FlatProperties p : list) {
String name = p.name();
//Work out type:
if(p.shapeLength() > 0){
if (p.shapeLength() > 0) {
//Array type
int[] shape = new int[p.shapeLength()];
for( int i=0; i<shape.length; i++ ){
for (int i = 0; i < shape.length; i++) {
shape[i] = p.shape(i);
}
// if(shape.length != 1){
@ -572,77 +582,77 @@ public class FlatBuffersMapper {
// throw new IllegalStateException("Multi-dimensional arrays not yet implemented");
// }
if(p.iLength() > 0){
if (p.iLength() > 0) {
int[] iArr = new int[p.iLength()];
for( int i=0; i<iArr.length; i++ ){
for (int i = 0; i < iArr.length; i++) {
iArr[i] = p.i(i);
}
if(shape.length == 0 || shape.length == 1) {
if (shape.length == 0 || shape.length == 1) {
out.put(name, iArr);
} else if(shape.length == 2){
} else if (shape.length == 2) {
out.put(name, ArrayUtil.reshapeInt(iArr, shape[0], shape[1]));
} else if(shape.length == 3){
} else if (shape.length == 3) {
out.put(name, ArrayUtil.reshapeInt(iArr, shape[0], shape[1], shape[2]));
}
} else if(p.dLength() > 0){
} else if (p.dLength() > 0) {
double[] dArr = new double[p.dLength()];
for( int i=0; i<dArr.length; i++ ){
for (int i = 0; i < dArr.length; i++) {
dArr[i] = p.d(i);
}
if(shape.length == 0 || shape.length == 1) {
if (shape.length == 0 || shape.length == 1) {
out.put(name, dArr);
} else if(shape.length == 2){
} else if (shape.length == 2) {
out.put(name, ArrayUtil.reshapeDouble(dArr, shape[0], shape[1]));
} else if(shape.length == 3){
} else if (shape.length == 3) {
out.put(name, ArrayUtil.reshapeDouble(dArr, shape[0], shape[1], shape[2]));
}
} else if(p.lLength() > 0) {
} else if (p.lLength() > 0) {
long[] lArr = new long[p.lLength()];
for (int i = 0; i < lArr.length; i++) {
lArr[i] = p.l(i);
}
if(shape.length == 0 || shape.length == 1) {
if (shape.length == 0 || shape.length == 1) {
out.put(name, lArr);
} else if(shape.length == 2){
} else if (shape.length == 2) {
out.put(name, ArrayUtil.reshapeLong(lArr, shape[0], shape[1]));
} else if(shape.length == 3){
} else if (shape.length == 3) {
out.put(name, ArrayUtil.reshapeLong(lArr, shape[0], shape[1], shape[2]));
}
} else if(p.bLength() > 0){
} else if (p.bLength() > 0) {
boolean[] bArr = new boolean[p.bLength()];
for( int i=0; i<bArr.length; i++ ){
for (int i = 0; i < bArr.length; i++) {
bArr[i] = p.b(i);
}
if(shape.length == 0 || shape.length == 1) {
if (shape.length == 0 || shape.length == 1) {
out.put(name, bArr);
} else if(shape.length == 2){
} else if (shape.length == 2) {
out.put(name, ArrayUtil.reshapeBoolean(bArr, shape[0], shape[1]));
} else if(shape.length == 3){
} else if (shape.length == 3) {
out.put(name, ArrayUtil.reshapeBoolean(bArr, shape[0], shape[1], shape[2]));
}
} else if(p.sLength() > 0){
} else if (p.sLength() > 0) {
String[] sArr = new String[p.sLength()];
for( int i=0; i<sArr.length; i++ ){
for (int i = 0; i < sArr.length; i++) {
sArr[i] = p.s(i);
}
if(shape.length == 0 || shape.length == 1) {
if (shape.length == 0 || shape.length == 1) {
out.put(name, sArr);
} else if(shape.length == 2){
} else if (shape.length == 2) {
out.put(name, ArrayUtil.reshapeObject(sArr, shape[0], shape[1]));
} else if(shape.length == 3){
} else if (shape.length == 3) {
out.put(name, ArrayUtil.reshapeObject(sArr, shape[0], shape[1], shape[2]));
}
} else if(p.aLength() > 0){
} else if (p.aLength() > 0) {
INDArray[] iArr = new INDArray[p.aLength()];
for( int i=0; i<iArr.length; i++ ){
for (int i = 0; i < iArr.length; i++) {
FlatArray fa = p.a(0);
iArr[i] = Nd4j.createFromFlatArray(fa);
}
if(shape.length == 0 || shape.length == 1) {
if (shape.length == 0 || shape.length == 1) {
out.put(name, iArr);
} else if(shape.length == 2){
} else if (shape.length == 2) {
out.put(name, ArrayUtil.reshapeObject(iArr, shape[0], shape[1]));
} else if(shape.length == 3){
} else if (shape.length == 3) {
out.put(name, ArrayUtil.reshapeObject(iArr, shape[0], shape[1], shape[2]));
}
} else {
@ -651,17 +661,17 @@ public class FlatBuffersMapper {
}
} else {
//non-array primitive, String or INDArray
if(p.bLength() > 0) {
if (p.bLength() > 0) {
out.put(name, p.b(0));
} else if(p.iLength() > 0){
} else if (p.iLength() > 0) {
out.put(name, p.i(0));
} else if(p.lLength() > 0){
} else if (p.lLength() > 0) {
out.put(name, p.l(0));
} else if(p.dLength() > 0){
} else if (p.dLength() > 0) {
out.put(name, p.d(0));
} else if(p.sLength() > 0){
} else if (p.sLength() > 0) {
out.put(name, p.s(0));
} else if(p.aLength() > 0){
} else if (p.aLength() > 0) {
FlatArray fa = p.a(0);
out.put(name, Nd4j.createFromFlatArray(fa));
} else {
@ -673,8 +683,8 @@ public class FlatBuffersMapper {
return out;
}
public static byte toVarType(VariableType variableType){
switch (variableType){
public static byte toVarType(VariableType variableType) {
switch (variableType) {
case VARIABLE:
return VarType.VARIABLE;
case CONSTANT:
@ -688,8 +698,8 @@ public class FlatBuffersMapper {
}
}
public static VariableType fromVarType(byte varType){
switch (varType){
public static VariableType fromVarType(byte varType) {
switch (varType) {
case VarType.VARIABLE:
return VariableType.VARIABLE;
case VarType.CONSTANT:

5
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/LegacyOpMapper.java
View File

@ -126,12 +126,7 @@ public class LegacyOpMapper {
case CONDITIONAL:
case LOOP:
case LOOP_COND:
case IF:
case RETURN:
case ENTER:
case EXIT:
case NEXT_ITERATION:
case MERGE:
default:
throw new UnsupportedOperationException("Unable to map op " + opNum + " of type " + opType);
}

19
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/converters/DifferentialFunctionClassHolder.java
View File

@ -25,6 +25,11 @@ import org.nd4j.imports.descriptors.onnx.OnnxDescriptorParser;
import org.nd4j.imports.descriptors.onnx.OpDescriptor;
import org.nd4j.imports.descriptors.tensorflow.TensorflowDescriptorParser;
import org.nd4j.linalg.api.ops.*;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Enter;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Exit;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Merge;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.NextIteration;
import org.nd4j.linalg.api.ops.impl.controlflow.compat.Switch;
import org.nd4j.linalg.api.ops.impl.layers.convolution.*;
import org.nd4j.linalg.exception.ND4JIllegalStateException;
import org.nd4j.linalg.factory.Nd4j;
@ -331,6 +336,18 @@ public class DifferentialFunctionClassHolder {
}
public Class<?> customOpClassForHashAndName(long customOpHash, String name){
switch (name) {
case Enter.OP_NAME:
return Enter.class;
case Exit.OP_NAME:
return Exit.class;
case NextIteration.OP_NAME:
return NextIteration.class;
case Merge.OP_NAME:
return Merge.class;
case Switch.OP_NAME:
return Switch.class;
default:
if(customOpHashToClasses.containsKey(customOpHash)){
return customOpHashToClasses.get(customOpHash).get(name);
} else if(customOpHashToClass.containsKey(customOpHash)){
@ -340,6 +357,8 @@ public class DifferentialFunctionClassHolder {
}
}
}
public static DifferentialFunctionClassHolder getInstance() {
return INSTANCE;
}

6
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/Op.java
View File

@ -69,14 +69,10 @@ public interface Op {
CONDITIONAL,
LOOP,
LOOP_COND,
IF,
RETURN,
ENTER,
EXIT,
NEXT_ITERATION,
RANDOM,
MERGE,
SUMMARYSTATS,
LOGIC
}
/**

31
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java
View File

@ -17,11 +17,13 @@
package org.nd4j.linalg.api.ops.impl.controlflow.compat;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.base.Preconditions;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.Op.Type;
import org.nd4j.linalg.api.shape.LongShapeDescriptor;
import org.tensorflow.framework.AttrValue;
import org.tensorflow.framework.GraphDef;
@ -32,13 +34,38 @@ import java.util.List;
import java.util.Map;
@Data
@NoArgsConstructor
public class Enter extends BaseCompatOp {
protected boolean isConstant;
public Enter(SameDiff sameDiff, SDVariable[] inputs){
super(sameDiff, inputs);
}
public Enter(SameDiff sameDiff, String frameName, SDVariable input){
super(sameDiff, new SDVariable[]{input});
this.frameName = frameName;
isConstant = input.isConstant();
}
public Enter(SameDiff sameDiff, String frameName, SDVariable input, boolean isConstant){
super(sameDiff, new SDVariable[]{input});
this.frameName = frameName;
this.isConstant = isConstant;
}
/**
* WARNING: do not change without changing serialization methods
* See {@link org.nd4j.autodiff.samediff.serde.FlatBuffersMapper#getOpNum(String, Type)}
* and {@link org.nd4j.imports.converters.DifferentialFunctionClassHolder#customOpClassForHashAndName(long, String)}
*/
public static final String OP_NAME = "enter";
public static final int OP_NUM = 100;
@Override
public String opName() {
return "enter";
return OP_NAME;
}
@Override
@ -62,7 +89,7 @@ public class Enter extends BaseCompatOp {
@Override
public Op.Type opType() {
return Op.Type.ENTER;
return Type.LOGIC;
}
@Override

20
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Exit.java
View File

@ -16,6 +16,7 @@
package org.nd4j.linalg.api.ops.impl.controlflow.compat;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.val;
import org.nd4j.autodiff.samediff.SDVariable;
@ -24,6 +25,7 @@ import org.nd4j.base.Preconditions;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.DynamicCustomOp;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.Op.Type;
import org.nd4j.linalg.api.shape.LongShapeDescriptor;
import org.tensorflow.framework.AttrValue;
import org.tensorflow.framework.GraphDef;
@ -34,10 +36,24 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
@NoArgsConstructor
public class Exit extends BaseCompatOp {
public Exit(SameDiff sameDiff, SDVariable x) {
super(sameDiff, new SDVariable[]{x});
}
/**
* WARNING: do not change without changing serialization methods
* See {@link org.nd4j.autodiff.samediff.serde.FlatBuffersMapper#getOpNum(String, Type)}
* and {@link org.nd4j.imports.converters.DifferentialFunctionClassHolder#customOpClassForHashAndName(long, String)}
*/
public static final String OP_NAME = "exit";
public static final int OP_NUM = 90;
@Override
public String opName() {
return "exit";
return OP_NAME;
}
@Override
@ -61,7 +77,7 @@ public class Exit extends BaseCompatOp {
@Override
public Op.Type opType() {
return Op.Type.EXIT;
return Type.LOGIC;
}
@Override

17
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Merge.java
View File

@ -21,6 +21,7 @@ import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.base.Preconditions;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.Op.Type;
import org.nd4j.linalg.api.shape.LongShapeDescriptor;
import org.tensorflow.framework.AttrValue;
import org.tensorflow.framework.GraphDef;
@ -41,9 +42,21 @@ public class Merge extends BaseCompatOp {
}
/**
* WARNING: do not change without changing serialization methods
* See {@link org.nd4j.autodiff.samediff.serde.FlatBuffersMapper#getOpNum(String, Type)}
* and {@link org.nd4j.imports.converters.DifferentialFunctionClassHolder#customOpClassForHashAndName(long, String)}
*/
public static final String OP_NAME = "merge";
public static final int OP_NUM = 60;
public Merge(SameDiff sd, SDVariable a, SDVariable b){
this(sd, new SDVariable[]{a, b});
}
@Override
public String opName() {
return "merge";
return OP_NAME;
}
@Override
@ -72,7 +85,7 @@ public class Merge extends BaseCompatOp {
@Override
public Op.Type opType() {
return Op.Type.MERGE;
return Type.LOGIC;
}
@Override

20
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/NextIteration.java
View File

@ -16,11 +16,13 @@
package org.nd4j.linalg.api.ops.impl.controlflow.compat;
import lombok.NoArgsConstructor;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.base.Preconditions;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.Op.Type;
import org.nd4j.linalg.api.shape.LongShapeDescriptor;
import org.tensorflow.framework.AttrValue;
import org.tensorflow.framework.GraphDef;
@ -31,10 +33,24 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
@NoArgsConstructor
public class NextIteration extends BaseCompatOp {
public NextIteration(SameDiff sameDiff, SDVariable x) {
super(sameDiff, new SDVariable[]{x});
}
/**
* WARNING: do not change without changing serialization methods
* See {@link org.nd4j.autodiff.samediff.serde.FlatBuffersMapper#getOpNum(String, Type)}
* and {@link org.nd4j.imports.converters.DifferentialFunctionClassHolder#customOpClassForHashAndName(long, String)}
*/
public static final String OP_NAME = "next_iteration";
public static final int OP_NUM = 80;
@Override
public String opName() {
return "next_iteration";
return OP_NAME;
}
@Override
@ -58,7 +74,7 @@ public class NextIteration extends BaseCompatOp {
@Override
public Op.Type opType() {
return Op.Type.NEXT_ITERATION;
return Type.LOGIC;
}
@Override

19
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Switch.java
View File

@ -16,12 +16,15 @@
package org.nd4j.linalg.api.ops.impl.controlflow.compat;
import com.google.common.collect.Lists;
import lombok.Getter;
import lombok.val;
import org.nd4j.autodiff.samediff.SDVariable;
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.base.Preconditions;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.Op.Type;
import org.nd4j.linalg.api.shape.LongShapeDescriptor;
import org.tensorflow.framework.AttrValue;
import org.tensorflow.framework.GraphDef;
@ -37,15 +40,27 @@ import java.util.Map;
*/
public class Switch extends BaseCompatOp {
@Getter
private SDVariable predicate;
public Switch(SameDiff sameDiff, SDVariable input, SDVariable predicate){
super(sameDiff, new SDVariable[]{input, predicate});
this.predicate = predicate;
}
public Switch(){ }
/**
* WARNING: do not change without changing serialization methods
* See {@link org.nd4j.autodiff.samediff.serde.FlatBuffersMapper#getOpNum(String, Type)}
* and {@link org.nd4j.imports.converters.DifferentialFunctionClassHolder#customOpClassForHashAndName(long, String)}
*/
public static final String OP_NAME = "switch";
public static final int OP_NUM = 30;
@Override
public String opName() {
return "switch";
return OP_NAME;
}
@Override
@ -72,7 +87,7 @@ public class Switch extends BaseCompatOp {
@Override
public Op.Type opType() {
return Op.Type.IF;
return Type.LOGIC;
}
@Override

14
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogSoftMax.java
View File

@ -39,6 +39,9 @@ import java.util.List;
*/
public class LogSoftMax extends DynamicCustomOp {
private Integer dimension = null;
public LogSoftMax(SameDiff sameDiff, SDVariable i_v) {
super(sameDiff, i_v);
}
@ -54,6 +57,12 @@ public class LogSoftMax extends DynamicCustomOp {
this(x, x);
}
public LogSoftMax(SameDiff sameDiff, SDVariable i_v, int dimension) {
this(sameDiff, i_v);
this.dimension = dimension;
addIArgument(dimension);
}
@Override
public String opName() {
@ -66,8 +75,13 @@ public class LogSoftMax extends DynamicCustomOp {
@Override
public List<SDVariable> doDiff(List<SDVariable> i_v) {
if(dimension == null) {
SDVariable ret = f().logSoftmaxDerivative(arg(), i_v.get(0));
return Collections.singletonList(ret);
} else {
SDVariable ret = f().logSoftmaxDerivative(arg(), i_v.get(0), dimension);
return Collections.singletonList(ret);
}
}
@Override

5
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LogSoftMaxDerivative.java
View File

@ -43,6 +43,11 @@ public class LogSoftMaxDerivative extends DynamicCustomOp {
super(null, new INDArray[]{in, gradO}, new INDArray[]{out});
}
public LogSoftMaxDerivative(SameDiff sameDiff, SDVariable arg, SDVariable wrt, int dimension) {
this(sameDiff, arg, wrt);
this.addIArgument(dimension);
}
/**
* The opName of this operation
*

35
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/samediff/NameScopeTests.java
View File

@ -129,4 +129,39 @@ public class NameScopeTests extends BaseNd4jTest {
}
}
}
@Test
public void testNoNesting(){
SameDiff SD = SameDiff.create();
SDVariable a = SD.constant(4);
NameScope scope = SD.withNameScope("test");
SDVariable out = SD.argmax(a);
out.add(45);
scope.close();
assertTrue("Var with name test/imax_1 exists", SD.variableMap().containsKey("test/imax_1"));
}
@Test
public void testNoTesting2(){
SameDiff SD = SameDiff.create();
SDVariable a = SD.constant(4);
SDVariable b = SD.constant(5).lt(4);
NameScope scope = SD.withNameScope("test");
SDVariable out = SD.f().switchOp(a, b)[0];
out.add(45);
scope.close();
assertTrue("Var with name test/switch:1 exists", SD.variableMap().containsKey("test/switch:1"));
}
}

330
nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/samediff/SameDiffTests.java
View File

@ -16,12 +16,30 @@
package org.nd4j.autodiff.samediff;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeNotNull;
import static org.nd4j.linalg.indexing.NDArrayIndex.all;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.nd4j.OpValidationSuite;
@ -43,7 +61,11 @@ import org.nd4j.linalg.api.ops.impl.shape.tensorops.TensorArray;
import org.nd4j.linalg.api.ops.impl.transforms.any.IsMax;
import org.nd4j.linalg.api.ops.impl.transforms.comparison.OldMax;
import org.nd4j.linalg.api.ops.impl.transforms.comparison.OldMin;
import org.nd4j.linalg.api.ops.impl.transforms.custom.*;
import org.nd4j.linalg.api.ops.impl.transforms.custom.GreaterThanOrEqual;
import org.nd4j.linalg.api.ops.impl.transforms.custom.IsNonDecreasing;
import org.nd4j.linalg.api.ops.impl.transforms.custom.IsNumericTensor;
import org.nd4j.linalg.api.ops.impl.transforms.custom.IsStrictlyIncreasing;
import org.nd4j.linalg.api.ops.impl.transforms.custom.LessThanOrEqual;
import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution;
import org.nd4j.linalg.api.shape.LongShapeDescriptor;
import org.nd4j.linalg.checkutil.NDArrayCreationUtil;
@ -53,9 +75,7 @@ import org.nd4j.linalg.dataset.adapter.SingletonMultiDataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.factory.Nd4jBackend;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.linalg.learning.GradientUpdater;
import org.nd4j.linalg.learning.config.Adam;
import org.nd4j.linalg.learning.config.Nesterovs;
import org.nd4j.linalg.ops.transforms.Transforms;
import org.nd4j.linalg.primitives.Pair;
import org.nd4j.nativeblas.NativeOpsHolder;
@ -63,29 +83,20 @@ import org.nd4j.weightinit.impl.OneInitScheme;
import org.nd4j.weightinit.impl.UniformInitScheme;
import org.nd4j.weightinit.impl.ZeroInitScheme;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.lang.reflect.Field;
import java.util.*;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeNotNull;
import static org.nd4j.linalg.indexing.NDArrayIndex.all;
/**
* Created by agibsonccc on 4/11/17.
*/
@Slf4j
public class SameDiffTests extends BaseNd4jTest {
private DataType initialType;
public SameDiffTests(Nd4jBackend b){
public SameDiffTests(Nd4jBackend b) {
super(b);
}
@Override
public char ordering(){
public char ordering() {
return 'c';
}
@ -317,7 +328,6 @@ public class SameDiffTests extends BaseNd4jTest {
SameDiff first = SameDiff.create();
SameDiff second = SameDiff.create();
SDVariable firstVar = first.var("one", new long[]{2, 2});
SDVariable secondVar = second.var(firstVar);
assertTrue(firstVar.getArr() == secondVar.getArr());
@ -330,7 +340,6 @@ public class SameDiffTests extends BaseNd4jTest {
SameDiff first = SameDiff.create();
SameDiff second = SameDiff.create();
SDVariable firstVar = first.var("one", new long[]{2, 2});
SDVariable secondVar = second.var(firstVar);
assumeNotNull(firstVar.getArr());
@ -418,7 +427,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
}, xAndY);
INDArray assertionForDiv = Nd4j.valueArrayOf(4, 4.0);
INDArray assertionForRDiv = Nd4j.valueArrayOf(4, 0.25);
assertEquals(assertionForDiv, sameDiff.getFunction("div").execAndEndResult());
@ -463,7 +471,8 @@ public class SameDiffTests extends BaseNd4jTest {
}, inputs);
INDArray assertion = sumInput.sum(1);
INDArray out = sameDiff.getFunction("sum").exec(Collections.emptyMap(), Collections.singletonList("sum")).get("sum");
INDArray out = sameDiff.getFunction("sum").exec(Collections.emptyMap(), Collections.singletonList("sum"))
.get("sum");
assertEquals(assertion, out);
}
@ -563,7 +572,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
}, inputVars);
//1 input plus 2 outputs
assertEquals(3, functionDef.variables().size());
@ -573,7 +581,8 @@ public class SameDiffTests extends BaseNd4jTest {
@Test
public void testIfStatementTrueBodyBackwards() {
OpValidationSuite.ignoreFailing(); //2019/01/14 AB: Disabled pending overhaul of SameDiff-defined conditional operations
OpValidationSuite
.ignoreFailing(); //2019/01/14 AB: Disabled pending overhaul of SameDiff-defined conditional operations
SameDiff sameDiff = SameDiff.create();
SameDiffFunctionDefinition conditionBody = new SameDiffFunctionDefinition() {
@Override
@ -584,7 +593,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
};
SameDiffFunctionDefinition trueBody = new SameDiffFunctionDefinition() {
@Override
public SDVariable[] define(SameDiff sameDiff, Map<String, INDArray> inputs, SDVariable[] variableInputs) {
@ -607,7 +615,6 @@ public class SameDiffTests extends BaseNd4jTest {
};
sameDiff.ifStatement(new DefaultSameDiffConditional(), conditionBody, trueBody, falseBody, firstInputs);
sameDiff.execBackwards(Collections.emptyMap());
SameDiff grad = sameDiff.getFunction("grad");
@ -625,7 +632,8 @@ public class SameDiffTests extends BaseNd4jTest {
@Test
public void testIfStatementTrueBody() {
OpValidationSuite.ignoreFailing(); //2019/01/14 AB: Disabled pending overhaul of SameDiff-defined conditional operations
OpValidationSuite
.ignoreFailing(); //2019/01/14 AB: Disabled pending overhaul of SameDiff-defined conditional operations
SameDiff sameDiff = SameDiff.create();
SameDiffFunctionDefinition conditionBody = new SameDiffFunctionDefinition() {
@ -637,7 +645,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
};
SameDiffFunctionDefinition trueBody = new SameDiffFunctionDefinition() {
@Override
public SDVariable[] define(SameDiff sameDiff, Map<String, INDArray> inputs, SDVariable[] variableInputs) {
@ -660,7 +667,6 @@ public class SameDiffTests extends BaseNd4jTest {
};
sameDiff.ifStatement(new DefaultSameDiffConditional(), conditionBody, trueBody, falseBody, firstInputs);
sameDiff.exec(Collections.emptyMap());
}
@ -668,7 +674,8 @@ public class SameDiffTests extends BaseNd4jTest {
@Test
public void testIfStatementFalseBody() {
OpValidationSuite.ignoreFailing(); //2019/01/14 AB: Disabled pending overhaul of SameDiff-defined conditional operations
OpValidationSuite
.ignoreFailing(); //2019/01/14 AB: Disabled pending overhaul of SameDiff-defined conditional operations
SameDiff sameDiff = SameDiff.create();
SameDiffFunctionDefinition conditionBody = new SameDiffFunctionDefinition() {
@ -680,7 +687,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
};
SameDiffFunctionDefinition trueBody = new SameDiffFunctionDefinition() {
@Override
public SDVariable[] define(SameDiff sameDiff, Map<String, INDArray> inputs, SDVariable[] variableInputs) {
@ -697,7 +703,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
};
//false body trigger
SDVariable[] secondInputs = new SDVariable[]{
sameDiff.setupFunction(sameDiff.var("two", new long[]{1, 1}))
@ -790,7 +795,6 @@ public class SameDiffTests extends BaseNd4jTest {
SDVariable weights = sd.var("W", new long[]{nIn, nOut});
SDVariable bias = sd.var("b", new long[]{1, nOut});
SDVariable mmul = sd.mmul("mmul", input, weights);
SDVariable z = mmul.add("z", bias);
SDVariable out = sd.math().tanh(z);
@ -888,7 +892,6 @@ public class SameDiffTests extends BaseNd4jTest {
val f = m.add(2.0);
val s = in2.add(5.0);
val arr = sd.execSingle(null, s.getVarName());
log.info("Result M: {}", m.getArr());
log.info("Result F: {}", f.getArr());
@ -939,7 +942,8 @@ public class SameDiffTests extends BaseNd4jTest {
val vector = Nd4j.linspace(1, 4, 4).reshape(4, 1);
val input1 = sd.var("input", matrix);
val input2 = sd.var("input2", vector);
val output = sd.mmul("output", input1, input2, MMulTranspose.builder().transposeA(true).transposeB(false).build());
val output = sd
.mmul("output", input1, input2, MMulTranspose.builder().transposeA(true).transposeB(false).build());
output.eval();
assertArrayEquals(new long[]{3, 1}, output.getShape());
}
@ -1026,12 +1030,11 @@ public class SameDiffTests extends BaseNd4jTest {
}
}, inputs);
SameDiff logisticGraph = sameDiffOuter.getFunction("oneminuspredictions");
Map<String, INDArray> inputsSubset = new HashMap<>();
inputsSubset.put("y", inputs.get("y"));
INDArray output = logisticGraph.exec(inputsSubset, Collections.singletonList("rsub")).get("rsub");
INDArray assertion = Nd4j.create(new double[]{0, 0, 1, 0}, new int[]{4,1});
INDArray assertion = Nd4j.create(new double[]{0, 0, 1, 0}, new int[]{4, 1});
assertEquals(assertion, output);
}
@ -1076,7 +1079,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
}, inputs);
SameDiff logisticPrediction = sameDiffOuter.getFunction("logisticPredictions");
List<String> logisticOpNameAssertions = Arrays.asList("mmul", "sigmoid");
@ -1146,7 +1148,8 @@ public class SameDiffTests extends BaseNd4jTest {
Activation.SOFTPLUS,
Activation.SOFTSIGN,
Activation.HARDTANH,
Activation.CUBE, //WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
Activation.CUBE,
//WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
Activation.RELU, //JVM crash
Activation.LEAKYRELU //JVM crash
};
@ -1289,8 +1292,9 @@ public class SameDiffTests extends BaseNd4jTest {
sd.exec(Collections.emptyMap(), sd.outputs());
for (int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++) {
assertEquals(1, out.getArr().get(all(), NDArrayIndex.point(i), all(), all()).getInt(0));
}
}
@ -1327,7 +1331,6 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray means = Nd4j.create(new float[]{2, 4}, new long[]{1, 2});
INDArray vars = Nd4j.create(new float[]{6, 8}, new long[]{1, 2});
SDVariable sdCounts = sd.var("counts", counts);
SDVariable sdMeans = sd.var("means", means);
SDVariable sdVars = sd.var("vars", vars);
@ -1363,7 +1366,6 @@ public class SameDiffTests extends BaseNd4jTest {
int imgH = 28;
int imgW = 28;
SameDiff sd = SameDiff.create();
INDArray depthWeightArr = Nd4j.create(kH, kW, nIn, depthWise);
@ -1720,7 +1722,6 @@ public class SameDiffTests extends BaseNd4jTest {
SDVariable in1 = sd.var("in1", ia);
SDVariable in2 = sd.var("in2", ib);
SDVariable t;
INDArray expOut;
switch (i) {
@ -1835,7 +1836,8 @@ public class SameDiffTests extends BaseNd4jTest {
val origShape = new long[]{3, 4};
for (int i = 0; i < 3; i++) {
for (Pair<INDArray, String> p : NDArrayCreationUtil.getAllTestMatricesWithShape(origShape[0], origShape[1], 12345, DataType.FLOAT)) {
for (Pair<INDArray, String> p : NDArrayCreationUtil
.getAllTestMatricesWithShape(origShape[0], origShape[1], 12345, DataType.FLOAT)) {
INDArray inArr = p.getFirst().muli(100);
SameDiff sd = SameDiff.create();
@ -1875,7 +1877,8 @@ public class SameDiffTests extends BaseNd4jTest {
val shape = origShape.clone();
shape[i] = 1;
for (Pair<INDArray, String> p : NDArrayCreationUtil.getAll3dTestArraysWithShape(12345, shape, DataType.FLOAT)) {
for (Pair<INDArray, String> p : NDArrayCreationUtil
.getAll3dTestArraysWithShape(12345, shape, DataType.FLOAT)) {
INDArray inArr = p.getFirst().muli(100);
SameDiff sd = SameDiff.create();
@ -1912,7 +1915,8 @@ public class SameDiffTests extends BaseNd4jTest {
val origShape = new long[]{3, 4};
for (int i = 0; i < 3; i++) {
for (Pair<INDArray, String> p : NDArrayCreationUtil.getAllTestMatricesWithShape(origShape[0], origShape[1], 12345, DataType.FLOAT)) {
for (Pair<INDArray, String> p : NDArrayCreationUtil
.getAllTestMatricesWithShape(origShape[0], origShape[1], 12345, DataType.FLOAT)) {
INDArray inArr = p.getFirst().muli(100);
SameDiff sd = SameDiff.create();
@ -1939,7 +1943,8 @@ public class SameDiffTests extends BaseNd4jTest {
val shape = origShape.clone();
shape[i] = 1;
for (Pair<INDArray, String> p : NDArrayCreationUtil.getAll3dTestArraysWithShape(12345, shape, DataType.FLOAT)) {
for (Pair<INDArray, String> p : NDArrayCreationUtil
.getAll3dTestArraysWithShape(12345, shape, DataType.FLOAT)) {
INDArray inArr = p.getFirst().muli(100);
SameDiff sd = SameDiff.create();
@ -2214,7 +2219,6 @@ public class SameDiffTests extends BaseNd4jTest {
SDVariable in = sd.var("in", 1, 2);
sd.associateArrayWithVariable(ia, in);
INDArray expFinite = Nd4j.create(new boolean[]{true, true});
SDVariable finite = sd.math().isFinite(in);
@ -2259,11 +2263,10 @@ public class SameDiffTests extends BaseNd4jTest {
SDVariable result2 = x.get(SDIndex.point(4), SDIndex.all());
assertEquals(expOut2, result2.eval());
INDArray expOut3 = arr.get(NDArrayIndex.interval(3, 8)).reshape(5,10);
INDArray expOut3 = arr.get(NDArrayIndex.interval(3, 8)).reshape(5, 10);
SDVariable result3 = x.get(SDIndex.interval(3, 8));
assertEquals(expOut3, result3.eval());
INDArray expOut4 = arr.get(NDArrayIndex.point(5), NDArrayIndex.interval(3, 8)).reshape(5);
SDVariable result4 = x.get(SDIndex.point(5), SDIndex.interval(3, 8));
assertEquals(expOut4, result4.eval());
@ -2295,7 +2298,6 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray s3a = s3.eval();
assertEquals(s3a, y3);
INDArray y4 = arr.get(NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.interval(3, 5));
SDVariable s4 = x.get(SDIndex.point(2), SDIndex.all(), SDIndex.interval(3, 5));
INDArray s4a = s4.eval();
@ -2409,7 +2411,6 @@ public class SameDiffTests extends BaseNd4jTest {
},
new int[]{3, 2, 4});
SDVariable x = sd.var(arr);
SDVariable result = sd.permute(x, 1, 0, 2);
assertEquals(expOut, result.eval());
@ -2470,7 +2471,7 @@ public class SameDiffTests extends BaseNd4jTest {
ExternalErrorsFunction fn = sd.f().externalErrors(out);
sd.execAndEndResult();
Map<String,INDArray> m = new HashMap<>();
Map<String, INDArray> m = new HashMap<>();
m.put("out-grad", externalGrad);
sd.execBackwards(m);
@ -2488,7 +2489,6 @@ public class SameDiffTests extends BaseNd4jTest {
assertEquals(externalGrad.mul(0.5), gradVar);
//Test model serialization:
}
@ -2620,7 +2620,7 @@ public class SameDiffTests extends BaseNd4jTest {
b.setArray(bA);
INDArray grad = Nd4j.linspace(1, 12, 12, DataType.FLOAT).reshape(3, 4);
Map<String,INDArray> phMap = new HashMap<>();
Map<String, INDArray> phMap = new HashMap<>();
phMap.put(fn.getGradPlaceholderName(), grad);
log.info("--------------- sd.execAndEndResult() ---------------");
@ -2723,7 +2723,6 @@ public class SameDiffTests extends BaseNd4jTest {
.build();
sd.setTrainingConfig(c);
sd.fit(new SingletonMultiDataSetIterator(new DataSet(inArr, null).toMultiDataSet()), 1);
INDArray out = tanh.eval();
@ -2757,7 +2756,7 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray inArr = Nd4j.rand(DataType.FLOAT, 1, 3);
in.setArray(inArr);
INDArray inArr2 = Nd4j.rand(DataType.FLOAT, 3,4);
INDArray inArr2 = Nd4j.rand(DataType.FLOAT, 3, 4);
TrainingConfig c = TrainingConfig.builder()
.updater(new Adam(0.1))
@ -2767,7 +2766,6 @@ public class SameDiffTests extends BaseNd4jTest {
.build();
sd.setTrainingConfig(c);
sd.fit(new SingletonMultiDataSetIterator(new MultiDataSet(new INDArray[]{inArr, inArr2}, null)), 1);
INDArray out = tanh.eval();
@ -2859,7 +2857,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
final INDArray out = Nd4j.concat(2, output).norm2();
SameDiff sd = SameDiff.create();
final SDVariable sdInput = sd.var("input", input);
@ -2905,7 +2902,6 @@ public class SameDiffTests extends BaseNd4jTest {
}
final INDArray out = Nd4j.concat(2, output).norm2();
SameDiff sd = SameDiff.create();
final SDVariable sdInput = sd.var("input", input);
@ -2917,13 +2913,11 @@ public class SameDiffTests extends BaseNd4jTest {
outputSlices[0] = x_0;
outputSlices[0] = sd.expandDims("X_0-e", outputSlices[0], 2);
final val x_1 = inputSlices[1];
outputSlices[1] = x_1;
outputSlices[1] = outputSlices[1].add(sd.squeeze("X_0-s", outputSlices[0], 2));
outputSlices[1] = sd.expandDims("X_1-e", outputSlices[1], 2);
SDVariable t = sd.concat(2, outputSlices);
t.norm2("out");
String err = OpValidation.validate(new TestCase(sd)
@ -3036,7 +3030,7 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testSameDiffBackprop1(){
public void testSameDiffBackprop1() {
SameDiff sd = SameDiff.create();
final SDVariable a = sd.var("a", Nd4j.rand(4, 4));
final SDVariable b = sd.var("b", Nd4j.rand(4, 4));
@ -3050,7 +3044,7 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testSameDiffNoGradForConstantAndPlaceholder(){
public void testSameDiffNoGradForConstantAndPlaceholder() {
SameDiff sd = SameDiff.create();
final SDVariable a = sd.var("a", Nd4j.rand(4, 4));
final SDVariable b = sd.constant("b", Nd4j.rand(4, 4));
@ -3058,16 +3052,16 @@ public class SameDiffTests extends BaseNd4jTest {
a.add(b.add(c)).sum().markAsLoss();
sd.execBackwards(Collections.singletonMap("c", Nd4j.rand(4,4 )));
sd.execBackwards(Collections.singletonMap("c", Nd4j.rand(4, 4)));
assertNotNull(sd.grad("a"));
assertNull(sd.grad("b"));
assertNull(sd.grad("c"));
}
@Test
public void testDuplicateNamePlaceholder(){
public void testDuplicateNamePlaceholder() {
for( int i=0; i<2; i++ ) {
for (int i = 0; i < 2; i++) {
SameDiff sd = SameDiff.create();
SDVariable x1 = i == 0 ? sd.placeHolder("a", DataType.FLOAT, 5, 3) : sd.var("a", DataType.FLOAT, 5, 3);
SDVariable x2 = i == 0 ? sd.placeHolder("b", DataType.FLOAT, 5, 3) : sd.var("b", DataType.FLOAT, 5, 3);
@ -3119,7 +3113,7 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testSameDiffGetArrayScalar(){
public void testSameDiffGetArrayScalar() {
final INDArray array = Nd4j.rand(1, 1);
final SameDiff sd = SameDiff.create();
final SDVariable a = sd.var("a", array.shape());
@ -3128,11 +3122,11 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testVariableRenaming(){
public void testVariableRenaming() {
SameDiff sd = SameDiff.create();
SDVariable v1 = sd.var("x", Nd4j.rand(DataType.FLOAT, 3,4));
SDVariable v2 = sd.var("y", Nd4j.rand(DataType.FLOAT, 4,5));
SDVariable v1 = sd.var("x", Nd4j.rand(DataType.FLOAT, 3, 4));
SDVariable v2 = sd.var("y", Nd4j.rand(DataType.FLOAT, 4, 5));
SDVariable v3 = v1.mmul("oldName", v2);
INDArray out = sd.execSingle(null, "oldName");
@ -3150,11 +3144,11 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testVariableRenaming2(){
public void testVariableRenaming2() {
SameDiff sd = SameDiff.create();
SDVariable v1 = sd.placeHolder("x", DataType.FLOAT,3,4);
SDVariable v2 = sd.var("y", Nd4j.rand(DataType.FLOAT, 4,5));
SDVariable v1 = sd.placeHolder("x", DataType.FLOAT, 3, 4);
SDVariable v2 = sd.var("y", Nd4j.rand(DataType.FLOAT, 4, 5));
SDVariable v3 = v1.mmul("oldName", v2);
SDVariable v4 = v3.std("out", false);
@ -3172,7 +3166,7 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testPlaceholderShapeValidation(){
public void testPlaceholderShapeValidation() {
SameDiff sd = SameDiff.create();
SDVariable ph1 = sd.placeHolder("ph1", DataType.FLOAT, 3, 4);
SDVariable ph2 = sd.placeHolder("ph2", DataType.FLOAT, -1, 4);
@ -3183,33 +3177,36 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray wrongShape = Nd4j.create(DataType.FLOAT, 2, 3);
INDArray wrongRank1 = Nd4j.create(DataType.FLOAT, 1);
INDArray wrongRank2 = Nd4j.create(DataType.FLOAT, 3, 4, 5);
for(SDVariable v : new SDVariable[]{ph1, ph2, ph3, ph4}){
for (SDVariable v : new SDVariable[]{ph1, ph2, ph3, ph4}) {
v.setArray(correctShape);
if(v != ph4) {
if (v != ph4) {
try {
v.setArray(wrongShape);
fail("Expected exception");
} catch (Exception t) {
String msg = t.getMessage();
assertTrue(msg, msg.contains("shape") && msg.contains("[2, 3]") && msg.contains(Arrays.toString(v.placeholderShape())));
assertTrue(msg, msg.contains("shape") && msg.contains("[2, 3]") && msg
.contains(Arrays.toString(v.placeholderShape())));
}
}
try{
try {
v.setArray(wrongRank1);
fail("Expected exception");
} catch (Exception t){
} catch (Exception t) {
String msg = t.getMessage();
assertTrue(msg, msg.contains("shape") && msg.contains("[1]") && msg.contains(Arrays.toString(v.placeholderShape())));
assertTrue(msg, msg.contains("shape") && msg.contains("[1]") && msg
.contains(Arrays.toString(v.placeholderShape())));
}
try{
try {
v.setArray(wrongRank2);
fail("Expected exception");
} catch (Exception t){
} catch (Exception t) {
String msg = t.getMessage();
assertTrue(msg, msg.contains("shape") && msg.contains("[3, 4, 5]") && msg.contains(Arrays.toString(v.placeholderShape())));
assertTrue(msg, msg.contains("shape") && msg.contains("[3, 4, 5]") && msg
.contains(Arrays.toString(v.placeholderShape())));
}
}
@ -3223,9 +3220,9 @@ public class SameDiffTests extends BaseNd4jTest {
.markLabelsUnused()
.updater(new Adam(1e-3)).build());
try{
try {
sd.fit(mds);
} catch (Exception t){
} catch (Exception t) {
String msg = t.getMessage();
assertTrue(msg, msg.contains("shape") && msg.contains("[2, 3]"));
}
@ -3233,7 +3230,7 @@ public class SameDiffTests extends BaseNd4jTest {
@Test
public void testInferenceWithoutLabel(){
public void testInferenceWithoutLabel() {
//We don't need a value for the label placeholder to calculate most values here
SameDiff sd = SameDiff.create();
@ -3252,15 +3249,14 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray inputArr = Nd4j.rand(DataType.FLOAT, minibatch, nIn);
Map<String,INDArray> m = sd.exec(Collections.singletonMap("in", inputArr), "softmax");
Map<String, INDArray> m = sd.exec(Collections.singletonMap("in", inputArr), "softmax");
assertEquals(1, m.size());
assertTrue(m.containsKey("softmax"));
INDArray out = m.get("softmax");
INDArray labelUnused = Nd4j.rand(DataType.FLOAT, minibatch, 3);
Map<String,INDArray> allPh = new HashMap<>();
Map<String, INDArray> allPh = new HashMap<>();
allPh.put("in", inputArr);
allPh.put("label", labelUnused);
m = sd.exec(allPh, "softmax");
@ -3271,7 +3267,7 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testInferenceWithoutUnnecessaryPlaceholders(){
public void testInferenceWithoutUnnecessaryPlaceholders() {
//We don't need an array for 2 of the placeholders to calculate the
SameDiff sd = SameDiff.create();
@ -3293,15 +3289,14 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray inputArr = Nd4j.rand(DataType.FLOAT, minibatch, nIn);
Map<String,INDArray> m = sd.exec(Collections.singletonMap("in", inputArr), "softmax");
Map<String, INDArray> m = sd.exec(Collections.singletonMap("in", inputArr), "softmax");
assertEquals(1, m.size());
assertTrue(m.containsKey("softmax"));
INDArray out = m.get("softmax");
INDArray labelUnused = Nd4j.rand(DataType.FLOAT, minibatch, 3);
Map<String,INDArray> allPh = new HashMap<>();
Map<String, INDArray> allPh = new HashMap<>();
allPh.put("in", inputArr);
allPh.put("label", labelUnused);
allPh.put("in2", Nd4j.scalar(1.0f));
@ -3314,7 +3309,7 @@ public class SameDiffTests extends BaseNd4jTest {
@Test
public void testConvertDTypes1(){
public void testConvertDTypes1() {
SameDiff sd = SameDiff.create();
SDVariable x = sd.var("x", Nd4j.rand(DataType.FLOAT, 3, 4));
@ -3329,15 +3324,15 @@ public class SameDiffTests extends BaseNd4jTest {
assertEquals(DataType.FLOAT, tanh.dataType());
assertEquals(DataType.FLOAT, stdev.dataType());
Map<String,INDArray> out = sd.exec(null, "x", "y", "z", "tanh", "stdev");
for(Map.Entry<String,INDArray> e : out.entrySet()){
Map<String, INDArray> out = sd.exec(null, "x", "y", "z", "tanh", "stdev");
for (Map.Entry<String, INDArray> e : out.entrySet()) {
assertEquals(e.getKey(), DataType.FLOAT, e.getValue().dataType());
}
assertEquals(DataType.FLOAT, x.getArr().dataType());
assertEquals(DataType.FLOAT, y.getArr().dataType());
Map<String,DataType> toConvert = new HashMap<>();
Map<String, DataType> toConvert = new HashMap<>();
toConvert.put("x", DataType.DOUBLE);
toConvert.put("y", DataType.DOUBLE);
sd.convertDataTypes(toConvert);
@ -3349,7 +3344,7 @@ public class SameDiffTests extends BaseNd4jTest {
assertEquals(DataType.DOUBLE, stdev.dataType());
out = sd.exec(null, "x", "y", "z", "tanh", "stdev");
for(Map.Entry<String,INDArray> e : out.entrySet()){
for (Map.Entry<String, INDArray> e : out.entrySet()) {
assertEquals(e.getKey(), DataType.DOUBLE, e.getValue().dataType());
}
@ -3358,7 +3353,7 @@ public class SameDiffTests extends BaseNd4jTest {
}
@Test
public void testConvertDTypes2(){
public void testConvertDTypes2() {
SameDiff sd = SameDiff.create();
SDVariable x = sd.placeHolder("x", DataType.FLOAT, 3, 4);
@ -3375,11 +3370,11 @@ public class SameDiffTests extends BaseNd4jTest {
assertEquals(DataType.DOUBLE, add.dataType());
assertEquals(DataType.DOUBLE, relu.dataType());
Map<String,INDArray> ph = Collections.singletonMap("x", Nd4j.rand(DataType.FLOAT, 3, 4));
Map<String, INDArray> ph = Collections.singletonMap("x", Nd4j.rand(DataType.FLOAT, 3, 4));
Map<String,INDArray> out = sd.exec(ph, "x", "y", "xD", "yD", "a", "r");
for(Map.Entry<String,INDArray> e : out.entrySet()){
if(e.getKey().equals("x") || e.getKey().equals("y")){
Map<String, INDArray> out = sd.exec(ph, "x", "y", "xD", "yD", "a", "r");
for (Map.Entry<String, INDArray> e : out.entrySet()) {
if (e.getKey().equals("x") || e.getKey().equals("y")) {
assertEquals(e.getKey(), DataType.FLOAT, e.getValue().dataType());
} else {
assertEquals(e.getKey(), DataType.DOUBLE, e.getValue().dataType());
@ -3388,7 +3383,7 @@ public class SameDiffTests extends BaseNd4jTest {
assertEquals(DataType.FLOAT, y.getArr().dataType());
Map<String,DataType> toConvert = new HashMap<>();
Map<String, DataType> toConvert = new HashMap<>();
toConvert.put("x", DataType.DOUBLE);
toConvert.put("y", DataType.DOUBLE);
sd.convertDataTypes(toConvert);
@ -3401,7 +3396,7 @@ public class SameDiffTests extends BaseNd4jTest {
assertEquals(DataType.DOUBLE, relu.dataType());
out = sd.exec(ph, "x", "y", "xD", "yD", "a", "r");
for(Map.Entry<String,INDArray> e : out.entrySet()){
for (Map.Entry<String, INDArray> e : out.entrySet()) {
assertEquals(e.getKey(), DataType.DOUBLE, e.getValue().dataType());
}
@ -3410,11 +3405,11 @@ public class SameDiffTests extends BaseNd4jTest {
@Test
public void testGradFnRequiredVars(){
public void testGradFnRequiredVars() {
//User can explicitly request that gradients for specific vars are available when differentiating (creating grad function),
// even if they normally wouldn't be needed or calculated
for(boolean reqPhVar : new boolean[]{false, true}){
for (boolean reqPhVar : new boolean[]{false, true}) {
// for(boolean reqPhVar : new boolean[]{true}){
SameDiff sd = SameDiff.create();
@ -3429,7 +3424,7 @@ public class SameDiffTests extends BaseNd4jTest {
INDArray in = Nd4j.rand(DataType.FLOAT, 1, 5);
if(reqPhVar){
if (reqPhVar) {
sd.createGradFunction("in");
assertNotNull(ph.gradient());
assertNotNull(w.gradient());
@ -3447,6 +3442,129 @@ public class SameDiffTests extends BaseNd4jTest {
}
}
@Test
public void testIf() throws IOException {
SameDiff SD = SameDiff.create();
SDVariable a = SD.placeHolder("a", DataType.DOUBLE);
SDVariable b = SD.var("b", Nd4j.createFromArray(5.0));
SDVariable c = SD.var("c", Nd4j.createFromArray(9.0));
SDVariable output = SD.ifCond("out", null, (sd) -> a.lt(b), (sd) -> c, (sd) -> c.add(5));
Map<String, INDArray> firstBranch = Maps.newHashMap();
firstBranch.put("a", Nd4j.createFromArray(3.0));
assertEquals(Nd4j.createFromArray(9.0), SD.exec(firstBranch, "out").get("out"));
Map<String, INDArray> secondBranch = Maps.newHashMap();
secondBranch.put("a", Nd4j.createFromArray(7.0));
assertEquals(Nd4j.createFromArray(14.0), SD.exec(secondBranch, "out").get("out"));
//TODO complains that it can't deserialize a meta type, but there are no meta type ops here
// looks like a difference between Op.Type and OpType. Switch is saved as a OpType.LOGIC
SD = SameDiff.fromFlatBuffers(SD.asFlatBuffers(false));
assertEquals(Nd4j.createFromArray(9.0), SD.exec(firstBranch, "out").get("out"));
assertEquals(Nd4j.createFromArray(14.0), SD.exec(secondBranch, "out").get("out"));
}
@Test
public void testNestedIf() throws IOException {
SameDiff SD = SameDiff.create();
SDVariable a = SD.var("a", Nd4j.createFromArray(2.0));
SDVariable b = SD.var("b", Nd4j.createFromArray(5.0));
SDVariable c = SD.var("c", Nd4j.createFromArray(9.0));
SDVariable d = SD.var("d", Nd4j.createFromArray(-7.0));
SDVariable output = SD.ifCond("out", null,
(sd) -> a.lt(b),
(sd) -> sd.ifCond(
(sd2) -> d.lte(0),
(sd2) -> c.add(1),
(sd2) -> d),
(sd) -> c.add(5));
INDArray out = output.eval();
assertEquals(Nd4j.createFromArray(10.0), out);
SD = SameDiff.fromFlatBuffers(SD.asFlatBuffers(false));
assertEquals(Nd4j.createFromArray(10.0), SD.exec(null, "out").get("out"));
}
@Test
public void testWhile() throws IOException {
SameDiff SD = SameDiff.create();
SDVariable countIn = SD.constant(5);
SDVariable sumIn = SD.constant(0);
SDVariable[] sum = SD.whileLoop("while_1", new SDVariable[]{countIn, sumIn},
(sd, vars) -> vars[0].gt(0),
(sd, vars) -> new SDVariable[]{vars[0].sub(1), vars[1].add(vars[0])});
INDArray out = sum[1].eval();
assertEquals(15, out.getInt(0));
String outName = sum[1].getVarName();
SD = SameDiff.fromFlatBuffers(SD.asFlatBuffers(false));
assertEquals(15, SD.exec(null, outName).get(outName).getInt(0));
}
@Test
@Ignore
public void testNestedWhile() throws IOException {
SameDiff SD = SameDiff.create();
SDVariable countIn = SD.constant(5);
SDVariable sumIn = SD.constant(0);
SDVariable sum2 = SD.constant(0);
//TODO creating constant instead of using sum2 causes errors
SDVariable[] sum = SD.whileLoop(new SDVariable[]{countIn, sumIn},
(sd, vars) -> vars[0].gt(0),
(sd, vars) -> new SDVariable[]{vars[0].sub(1),
vars[1].add(sd.whileLoop(new SDVariable[]{vars[0], sum2},
(sd2, vars2) -> vars2[0].gt(0),
(sd2, vars2) -> new SDVariable[]{vars2[0].sub(1), vars2[1].add(vars2[0])})[1])});
INDArray out = sum[1].eval();
assertEquals(35, out.getInt(0));
String outName = sum[1].getVarName();
SD = SameDiff.fromFlatBuffers(SD.asFlatBuffers(false));
assertEquals(35, SD.exec(null, outName).get(outName).getInt(0));
}
@Test
public void testNestedWhileIf() throws IOException {
SameDiff SD = SameDiff.create();
SDVariable countIn = SD.constant(5);
SDVariable sumIn = SD.constant(0);
SDVariable hundred = SD.constant(100);
SDVariable[] sum = SD.whileLoop(new SDVariable[]{countIn, sumIn},
(sd, vars) -> vars[0].gte(0),
(sd, vars) -> new SDVariable[]{vars[0].sub(1), vars[1].add(
sd.ifCond((sd2) -> vars[0].eq(0),
(sd2) -> vars[0].add(100), //TODO replace with hundred and things break
(sd2) -> vars[0])
)});
INDArray out = sum[1].eval();
assertEquals(115, out.getInt(0));
String outName = sum[1].getVarName();
SD = SameDiff.fromFlatBuffers(SD.asFlatBuffers(false));
assertEquals(115, SD.exec(null, outName).get(outName).getInt(0));
}
}