Assorted fixes (#318)

* #8777 MultiLayerNetwork.evaluate(MultiDataSetIterator) overload

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8768 SameDiff.equals

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8750 shade freemarker library and switch to it in DL4J UI

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8704 DL4J UI redirect

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8776 RecordReaderDataSetIterator builder collectMetaData fix

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8718 Fix DL4J doEvaluation metadata

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8715 ArchiveUtils - Add option to not log every extracted file

Signed-off-by: Alex Black <blacka101@gmail.com>

* No exception for evaluations that don't support metadata

Signed-off-by: Alex Black <blacka101@gmail.com>

* Fixes

Signed-off-by: Alex Black <blacka101@gmail.com>

* #8765 CompGraph+MDS fix for SharedTrainingMaster

Signed-off-by: Alex Black <blacka101@gmail.com>

* small fix

Signed-off-by: Alex Black <blacka101@gmail.com>

* Timeout

Signed-off-by: Alex Black <blacka101@gmail.com>

* Ignore

Signed-off-by: Alex Black <blacka101@gmail.com>

* Revert freemarker shading

Signed-off-by: Alex Black <blacka101@gmail.com>

* Ignore

Signed-off-by: Alex Black <blacka101@gmail.com>

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetiteratorTest.java

@@ -1381,4 +1381,17 @@ public class RecordReaderDataSetiteratorTest extends BaseDL4JTest {
         assertNotNull(ds.getFeatures());
         assertNull(ds.getLabels());
     }
+
+
+    @Test
+    public void testCollectMetaData(){
+        RecordReaderDataSetIterator trainIter = new RecordReaderDataSetIterator.Builder(new CollectionRecordReader(Collections.<List<Writable>>emptyList()), 1)
+                .collectMetaData(true)
+                .build();
+        assertTrue(trainIter.isCollectMetaData());
+        trainIter.setCollectMetaData(false);
+        assertFalse(trainIter.isCollectMetaData());
+        trainIter.setCollectMetaData(true);
+        assertTrue(trainIter.isCollectMetaData());
+    }
 }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalTest.java

@@ -33,7 +33,6 @@ import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator;
 import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator;
 import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator;
 import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator;
-import org.deeplearning4j.eval.meta.Prediction;
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
 import org.deeplearning4j.nn.conf.*;
 import org.deeplearning4j.nn.conf.layers.*;
@@ -52,19 +51,13 @@ import org.nd4j.linalg.dataset.api.MultiDataSet;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
 import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize;
 import org.nd4j.linalg.factory.Nd4j;
-import org.nd4j.linalg.indexing.INDArrayIndex;
-import org.nd4j.linalg.indexing.NDArrayIndex;
-import org.nd4j.linalg.io.ClassPathResource;
 import org.nd4j.linalg.learning.config.Sgd;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
-import org.nd4j.linalg.util.FeatureUtil;
 import org.nd4j.resources.Resources;
 
 import java.util.*;
 
 import static org.junit.Assert.*;
-import static org.nd4j.linalg.indexing.NDArrayIndex.all;
-import static org.nd4j.linalg.indexing.NDArrayIndex.interval;
 
 /**
  * Created by agibsonccc on 12/22/14.
@@ -165,7 +158,7 @@ public class EvalTest extends BaseDL4JTest {
         assertEquals(evalExpected.getConfusionMatrix(), evalActual.getConfusionMatrix());
     }
 
-    @Test(timeout = 300000)
+    @Test
     public void testEvaluationWithMetaData() throws Exception {
 
         RecordReader csv = new CSVRecordReader();
@@ -256,6 +249,30 @@ public class EvalTest extends BaseDL4JTest {
             assertEquals(actualCounts[i], actualClassI.size());
             assertEquals(predictedCounts[i], predictedClassI.size());
         }
+
+
+        //Finally: test doEvaluation methods
+        rrdsi.reset();
+        org.nd4j.evaluation.classification.Evaluation e2 = new org.nd4j.evaluation.classification.Evaluation();
+        net.doEvaluation(rrdsi, e2);
+        for (int i = 0; i < 3; i++) {
+            List<org.nd4j.evaluation.meta.Prediction> actualClassI = e2.getPredictionsByActualClass(i);
+            List<org.nd4j.evaluation.meta.Prediction> predictedClassI = e2.getPredictionByPredictedClass(i);
+            assertEquals(actualCounts[i], actualClassI.size());
+            assertEquals(predictedCounts[i], predictedClassI.size());
+        }
+
+        ComputationGraph cg = net.toComputationGraph();
+        rrdsi.reset();
+        e2 = new org.nd4j.evaluation.classification.Evaluation();
+        cg.doEvaluation(rrdsi, e2);
+        for (int i = 0; i < 3; i++) {
+            List<org.nd4j.evaluation.meta.Prediction> actualClassI = e2.getPredictionsByActualClass(i);
+            List<org.nd4j.evaluation.meta.Prediction> predictedClassI = e2.getPredictionByPredictedClass(i);
+            assertEquals(actualCounts[i], actualClassI.size());
+            assertEquals(predictedCounts[i], predictedClassI.size());
+        }
+
     }
 
     private static void apply(org.nd4j.evaluation.classification.Evaluation e, int nTimes, INDArray predicted, INDArray actual) {
@@ -504,11 +521,11 @@ public class EvalTest extends BaseDL4JTest {
             list.add(new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{ds.getFeatures()}, new INDArray[]{ds.getLabels(), ds.getLabels()}));
         }
 
-        Evaluation e = new Evaluation();
+        org.nd4j.evaluation.classification.Evaluation e = new org.nd4j.evaluation.classification.Evaluation();
-        RegressionEvaluation e2 = new RegressionEvaluation();
+        org.nd4j.evaluation.regression.RegressionEvaluation e2 = new org.nd4j.evaluation.regression.RegressionEvaluation();
-        Map<Integer,IEvaluation[]> evals = new HashMap<>();
+        Map<Integer,org.nd4j.evaluation.IEvaluation[]> evals = new HashMap<>();
-        evals.put(0, new IEvaluation[]{(IEvaluation) e});
+        evals.put(0, new org.nd4j.evaluation.IEvaluation[]{e});
-        evals.put(1, new IEvaluation[]{(IEvaluation) e2});
+        evals.put(1, new org.nd4j.evaluation.IEvaluation[]{e2});
 
         cg.evaluate(new IteratorMultiDataSetIterator(list.iterator(), 30), evals);
 
@@ -567,14 +584,14 @@ public class EvalTest extends BaseDL4JTest {
         }
 
         try {
-            net.evaluateROC(iter);
+            net.evaluateROC(iter, 0);
             fail("Expected exception");
         } catch (IllegalStateException e){
             assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC"));
         }
 
         try {
-            net.evaluateROCMultiClass(iter);
+            net.evaluateROCMultiClass(iter, 0);
             fail("Expected exception");
         } catch (IllegalStateException e){
             assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass"));
@@ -589,14 +606,14 @@ public class EvalTest extends BaseDL4JTest {
         }
 
         try {
-            cg.evaluateROC(iter);
+            cg.evaluateROC(iter, 0);
             fail("Expected exception");
         } catch (IllegalStateException e){
             assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC"));
         }
 
         try {
-            cg.evaluateROCMultiClass(iter);
+            cg.evaluateROCMultiClass(iter, 0);
             fail("Expected exception");
         } catch (IllegalStateException e){
             assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass"));
@@ -606,10 +623,10 @@ public class EvalTest extends BaseDL4JTest {
         //Disable validation, and check same thing:
         net.getLayerWiseConfigurations().setValidateOutputLayerConfig(false);
         net.evaluate(iter);
-        net.evaluateROCMultiClass(iter);
+        net.evaluateROCMultiClass(iter, 0);
 
         cg.getConfiguration().setValidateOutputLayerConfig(false);
         cg.evaluate(iter);
-        cg.evaluateROCMultiClass(iter);
+        cg.evaluateROCMultiClass(iter, 0);
     }
 }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java

@@ -61,7 +61,7 @@ public class RegressionEvalTest extends BaseDL4JTest {
 
         DataSet ds = new DataSet(f, l);
         DataSetIterator iter = new ExistingDataSetIterator(Collections.singletonList(ds));
-        RegressionEvaluation re = net.evaluateRegression(iter);
+        org.nd4j.evaluation.regression.RegressionEvaluation re = net.evaluateRegression(iter);
 
         for (int i = 0; i < 5; i++) {
             assertEquals(1.0, re.meanSquaredError(i), 1e-6);

deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java

@@ -86,7 +86,7 @@ public abstract class CacheableExtractableDataSetFetcher implements CacheableDat
         }
 
         try {
-            ArchiveUtils.unzipFileTo(tmpFile.getAbsolutePath(), localCacheDir.getAbsolutePath());
+            ArchiveUtils.unzipFileTo(tmpFile.getAbsolutePath(), localCacheDir.getAbsolutePath(), false);
         } catch (Throwable t){
             //Catch any errors during extraction, and delete the directory to avoid leaving the dir in an invalid state
             if(localCacheDir.exists())

deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java

@@ -205,6 +205,7 @@ public class RecordReaderDataSetIterator implements DataSetIterator {
         this.numPossibleLabels = b.numPossibleLabels;
         this.regression = b.regression;
         this.preProcessor = b.preProcessor;
+        this.collectMetaData = b.collectMetaData;
     }
 
     /**

deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-japanese/src/main/java/com/atilika/kuromoji/util/KuromojiBinFilesFetcher.java

@@ -67,7 +67,7 @@ public class KuromojiBinFilesFetcher {
                             new URL("https://dl4jdata.blob.core.windows.net/kuromoji/kuromoji_bin_files.tar.gz"),
                             tarFile);
         }
-        ArchiveUtils.unzipFileTo(tarFile.getAbsolutePath(), rootDir.getAbsolutePath());
+        ArchiveUtils.unzipFileTo(tarFile.getAbsolutePath(), rootDir.getAbsolutePath(), false);
 
         return rootDir.getAbsoluteFile();
     }

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java

@@ -4170,6 +4170,7 @@ public class ComputationGraph implements Serializable, Model, NeuralNetwork {
                 INDArray[] featuresMasks = next.getFeaturesMaskArrays();
                 INDArray[] labels = next.getLabels();
                 INDArray[] labelMasks = next.getLabelsMaskArrays();
+                List<Serializable> meta = next.getExampleMetaData();
 
                 try (MemoryWorkspace ws = outputWs.notifyScopeEntered()) {
                     INDArray[] out = outputOfLayersDetached(false, FwdPassType.STANDARD, getOutputLayerIndices(), features, featuresMasks, labelMasks, true, false, ws);
@@ -4188,7 +4189,7 @@ public class ComputationGraph implements Serializable, Model, NeuralNetwork {
 
                         try (MemoryWorkspace wsO = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
                             for (IEvaluation evaluation : evalsThisOutput)
-                                evaluation.eval(currLabel, currOut, next.getLabelsMaskArray(i));
+                                evaluation.eval(currLabel, currOut, next.getLabelsMaskArray(i), meta);
                         }
                     }
                 }

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/util/ComputationGraphUtil.java

@@ -23,6 +23,9 @@ import org.nd4j.linalg.dataset.api.MultiDataSet;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
 import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator;
 
+import java.io.Serializable;
+import java.util.List;
+
 public class ComputationGraphUtil {
 
     private ComputationGraphUtil() {}
@@ -33,13 +36,16 @@ public class ComputationGraphUtil {
         INDArray l = dataSet.getLabels();
         INDArray fMask = dataSet.getFeaturesMaskArray();
         INDArray lMask = dataSet.getLabelsMaskArray();
+        List<Serializable> meta = dataSet.getExampleMetaData();
 
         INDArray[] fNew = f == null ? null : new INDArray[] {f};
         INDArray[] lNew = l == null ? null : new INDArray[] {l};
         INDArray[] fMaskNew = (fMask != null ? new INDArray[] {fMask} : null);
         INDArray[] lMaskNew = (lMask != null ? new INDArray[] {lMask} : null);
 
-        return new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
+        org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
+        mds.setExampleMetaData(meta);
+        return mds;
     }
 
     /** Convert a DataSetIterator to a MultiDataSetIterator, via an adaptor class */

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java

@@ -25,14 +25,11 @@ import lombok.val;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.bytedeco.javacpp.Pointer;
-import org.nd4j.adapters.OutputAdapter;
-import org.nd4j.linalg.dataset.AsyncDataSetIterator;;
 import org.deeplearning4j.datasets.iterator.MultiDataSetWrapperIterator;
-import org.deeplearning4j.eval.RegressionEvaluation;
 import org.deeplearning4j.exception.DL4JException;
 import org.deeplearning4j.exception.DL4JInvalidInputException;
-import org.deeplearning4j.nn.api.*;
 import org.deeplearning4j.nn.api.Updater;
+import org.deeplearning4j.nn.api.*;
 import org.deeplearning4j.nn.api.layers.IOutputLayer;
 import org.deeplearning4j.nn.api.layers.RecurrentLayer;
 import org.deeplearning4j.nn.conf.*;
@@ -44,8 +41,8 @@ import org.deeplearning4j.nn.gradient.Gradient;
 import org.deeplearning4j.nn.graph.ComputationGraph;
 import org.deeplearning4j.nn.layers.FrozenLayer;
 import org.deeplearning4j.nn.layers.FrozenLayerWithBackprop;
-import org.deeplearning4j.nn.layers.recurrent.BidirectionalLayer;
 import org.deeplearning4j.nn.layers.LayerHelper;
+import org.deeplearning4j.nn.layers.recurrent.BidirectionalLayer;
 import org.deeplearning4j.nn.layers.wrapper.BaseWrapperLayer;
 import org.deeplearning4j.nn.updater.UpdaterCreator;
 import org.deeplearning4j.nn.workspace.ArrayType;
@@ -58,19 +55,23 @@ import org.deeplearning4j.util.CrashReportingUtil;
 import org.deeplearning4j.util.ModelSerializer;
 import org.deeplearning4j.util.NetworkUtils;
 import org.deeplearning4j.util.OutputLayerUtil;
+import org.nd4j.adapters.OutputAdapter;
 import org.nd4j.base.Preconditions;
 import org.nd4j.evaluation.IEvaluation;
 import org.nd4j.evaluation.classification.Evaluation;
 import org.nd4j.evaluation.classification.ROC;
 import org.nd4j.evaluation.classification.ROCMultiClass;
+import org.nd4j.evaluation.regression.RegressionEvaluation;
 import org.nd4j.linalg.api.buffer.DataType;
 import org.nd4j.linalg.api.memory.MemoryWorkspace;
+import org.nd4j.linalg.api.memory.abstracts.DummyWorkspace;
 import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration;
 import org.nd4j.linalg.api.memory.enums.AllocationPolicy;
 import org.nd4j.linalg.api.memory.enums.LearningPolicy;
 import org.nd4j.linalg.api.memory.enums.ResetPolicy;
 import org.nd4j.linalg.api.memory.enums.SpillPolicy;
 import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.AsyncDataSetIterator;
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.dataset.api.MultiDataSet;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
@@ -84,7 +85,6 @@ import org.nd4j.linalg.heartbeat.reports.Task;
 import org.nd4j.linalg.heartbeat.utils.EnvironmentUtils;
 import org.nd4j.linalg.heartbeat.utils.TaskUtils;
 import org.nd4j.linalg.indexing.NDArrayIndex;
-import org.nd4j.linalg.api.memory.abstracts.DummyWorkspace;
 import org.nd4j.linalg.primitives.Pair;
 import org.nd4j.linalg.primitives.Triple;
 import org.nd4j.linalg.schedule.ISchedule;
@@ -96,6 +96,8 @@ import org.nd4j.util.OneTimeLogger;
 import java.io.*;
 import java.util.*;
 
+;
+
 
 /**
  * MultiLayerNetwork is a neural network with multiple layers in a stack, and usually an output layer.<br>
@@ -3315,19 +3317,39 @@ public class MultiLayerNetwork implements Serializable, Classifier, Layer, Neura
      * @param iterator Iterator to evaluate on
      * @return Evaluation object; results of evaluation on all examples in the data set
      */
-    public <T extends Evaluation> T evaluate(DataSetIterator iterator) {
+    public <T extends Evaluation> T evaluate(@NonNull DataSetIterator iterator) {
         return (T)evaluate(iterator, null);
     }
 
+    /**
+     * Evaluate the network (classification performance).
+     * Can only be used with MultiDataSetIterator instances with a single input/output array
+     *
+     * @param iterator Iterator to evaluate on
+     * @return Evaluation object; results of evaluation on all examples in the data set
+     */
+    public Evaluation evaluate(@NonNull MultiDataSetIterator iterator) {
+        return evaluate(new MultiDataSetWrapperIterator(iterator));
+    }
+
     /**
      * Evaluate the network for regression performance
      * @param iterator Data to evaluate on
-     * @return
+     * @return Regression evaluation
      */
     public <T extends RegressionEvaluation> T evaluateRegression(DataSetIterator iterator) {
         return (T)doEvaluation(iterator, new RegressionEvaluation(iterator.totalOutcomes()))[0];
     }
 
+    /**
+     * Evaluate the network for regression performance
+     * Can only be used with MultiDataSetIterator instances with a single input/output array
+     * @param iterator Data to evaluate on
+     */
+    public org.nd4j.evaluation.regression.RegressionEvaluation evaluateRegression(MultiDataSetIterator iterator) {
+        return evaluateRegression(new MultiDataSetWrapperIterator(iterator));
+    }
+
     /**
      * @deprecated To be removed - use {@link #evaluateROC(DataSetIterator, int)} to enforce selection of appropriate ROC/threshold configuration
      */
@@ -3424,6 +3446,7 @@ public class MultiLayerNetwork implements Serializable, Classifier, Layer, Neura
             INDArray labels = next.getLabels();
             INDArray fMask = next.getFeaturesMaskArray();
             INDArray lMask = next.getLabelsMaskArray();
+            List<Serializable> meta = next.getExampleMetaData();
 
 
             if (!useRnnSegments) {
@@ -3433,7 +3456,7 @@ public class MultiLayerNetwork implements Serializable, Classifier, Layer, Neura
 
                     try (MemoryWorkspace wsO = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
                         for (T evaluation : evaluations)
-                            evaluation.eval(labels, out, lMask);
+                            evaluation.eval(labels, out, lMask, meta);
                     }
                 }
             } else {

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/AdaptiveThresholdAlgorithm.java

@@ -222,8 +222,11 @@ public class AdaptiveThresholdAlgorithm implements ThresholdAlgorithm {
             if(a == null || Double.isNaN(a.lastThreshold))
                 return;
 
+
             lastThresholdSum += a.lastThreshold;
+            if (!Double.isNaN(a.lastSparsity)) {
                 lastSparsitySum += a.lastSparsity;
+            }
             count++;
         }
 

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/pom.xml

@@ -38,16 +38,22 @@
             <artifactId>nd4j-aeron</artifactId>
             <version>${nd4j.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.nd4j</groupId>
-            <artifactId>nd4j-parameter-server-node_2.11</artifactId>
-            <version>${nd4j.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.deeplearning4j</groupId>
             <artifactId>dl4j-spark_2.11</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.nd4j</groupId>
+            <artifactId>nd4j-parameter-server-node_2.11</artifactId>
+            <version>${nd4j.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>net.jpountz.lz4</groupId>
+                    <artifactId>lz4</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualMultiDataSetIterator.java

@@ -23,6 +23,7 @@ import org.nd4j.linalg.dataset.api.iterator.ParallelMultiDataSetIterator;
 
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * This MultiDataSetIterator implementation does accumulation of MultiDataSets from different Spark executors, wrt Thread/Device Affinity
@@ -32,14 +33,16 @@ import java.util.List;
 public class VirtualMultiDataSetIterator implements ParallelMultiDataSetIterator {
 
     protected final List<Iterator<MultiDataSet>> iterators;
+    protected final AtomicInteger position;
 
     public VirtualMultiDataSetIterator(@NonNull List<Iterator<MultiDataSet>> iterators) {
         this.iterators = iterators;
+        this.position = new AtomicInteger(0);
     }
 
     @Override
     public MultiDataSet next(int num) {
-        return null;
+        return next();
     }
 
     @Override
@@ -59,27 +62,34 @@ public class VirtualMultiDataSetIterator implements ParallelMultiDataSetIterator
 
     @Override
     public boolean asyncSupported() {
-        return false;
+        return true;
     }
 
     @Override
     public void reset() {
-
+        throw new UnsupportedOperationException();
     }
 
     @Override
     public boolean hasNext() {
-        return false;
+        // just checking if that's not the last iterator, or if that's the last one - check if it has something
+        boolean ret = position.get() < iterators.size() - 1
+                || (position.get() < iterators.size() && iterators.get(position.get()).hasNext());
+        return ret;
     }
 
     @Override
     public MultiDataSet next() {
-        return null;
+        // TODO: this solution isn't ideal, it assumes non-empty iterators all the time. Would be nice to do something here
+        if (!iterators.get(position.get()).hasNext())
+            position.getAndIncrement();
+
+        return iterators.get(position.get()).next();
     }
 
     @Override
     public void remove() {
-
+        // no-op
     }
 
     @Override

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/pw/SharedTrainingWrapper.java

@@ -109,6 +109,7 @@ public class SharedTrainingWrapper {
 
         // now we're creating DataSetIterators, to feed ParallelWrapper
         iteratorDS = new VirtualDataSetIterator(iteratorsDS);
+        iteratorMDS = new VirtualMultiDataSetIterator(iteratorsMDS);
     }
 
     public static synchronized SharedTrainingWrapper getInstance(long id) {
@@ -447,17 +448,19 @@ public class SharedTrainingWrapper {
                 throw new DL4JInvalidConfigException("No iterators were defined for training");
 
             try {
-                while((iteratorDS != null && iteratorDS.hasNext()) || (iteratorMDS != null && iteratorMDS.hasNext())) {
+                boolean dsNext;
+                boolean mdsNext;
+                while((dsNext = iteratorDS != null && iteratorDS.hasNext()) || (mdsNext = iteratorMDS != null && iteratorMDS.hasNext())) {
                     //Loop as a guard against concurrent modifications and RCs
 
                     if (wrapper != null) {
-                        if (iteratorDS != null)
+                        if (dsNext)
                             wrapper.fit(iteratorDS);
                         else
                             wrapper.fit(iteratorMDS);
                     } else {
                         // if wrapper is null, we're fitting standalone model then
-                        if (iteratorDS != null) {
+                        if (dsNext) {
                             if (model instanceof ComputationGraph) {
                                 ((ComputationGraph) originalModel).fit(iteratorDS);
                             } else if (model instanceof MultiLayerNetwork) {
@@ -472,6 +475,7 @@ public class SharedTrainingWrapper {
                         }
                     }
 
+                    if(consumer != null)
                         consumer.getUpdatesQueue().purge();
                 }
             } catch (Throwable t){

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java

@@ -116,8 +116,7 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable
     }
 
     protected int numExecutors() {
-        int numProc = Runtime.getRuntime().availableProcessors();
+        return 4;
-        return Math.min(4, numProc);
     }
 
     protected MultiLayerConfiguration getBasicConf() {

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java

@@ -49,6 +49,7 @@ import org.junit.rules.TemporaryFolder;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.MultiDataSet;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
 import org.nd4j.linalg.indexing.NDArrayIndex;
 import org.nd4j.linalg.learning.config.AMSGrad;
@@ -66,20 +67,26 @@ import java.util.concurrent.ConcurrentHashMap;
 import static org.junit.Assert.*;
 
 @Slf4j
-@Ignore("AB 2019/05/21 - Failing - Issue #7657")
+//@Ignore("AB 2019/05/21 - Failing - Issue #7657")
 public class GradientSharingTrainingTest extends BaseSparkTest {
 
     @Rule
     public TemporaryFolder testDir = new TemporaryFolder();
 
+    @Override
+    public long getTimeoutMilliseconds() {
+        return 90000L;
+    }
+
     @Test
     public void trainSanityCheck() throws Exception {
 
+        for(boolean mds : new boolean[]{false, true}) {
             INDArray last = null;
             INDArray lastDup = null;
             for (String s : new String[]{"paths", "direct", "export"}) {
                 System.out.println("--------------------------------------------------------------------------------------------------------------");
-            log.info("Starting: {}", s);
+                log.info("Starting: {} - {}", s, (mds ? "MultiDataSet" : "DataSet"));
                 boolean isPaths = "paths".equals(s);
 
                 RDDTrainingApproach rddTrainingApproach;
@@ -144,7 +151,11 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
                     DataSet d = iter.next();
                     if (isPaths) {
                         File out = new File(f, count + ".bin");
+                        if(mds){
+                            d.toMultiDataSet().save(out);
+                        } else {
                             d.save(out);
+                        }
                         String path = "file:///" + out.getAbsolutePath().replaceAll("\\\\", "/");
                         paths.add(path);
                     }
@@ -160,6 +171,27 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
 
                     INDArray paramsBefore = sparkNet.getNetwork().params().dup();
                     ComputationGraph after;
+                    if(mds) {
+                        //Fitting from MultiDataSet
+                        List<MultiDataSet> mdsList = new ArrayList<>();
+                        for(DataSet d : ds){
+                            mdsList.add(d.toMultiDataSet());
+                        }
+                        switch (s) {
+                            case "direct":
+                            case "export":
+                                JavaRDD<MultiDataSet> dsRDD = sc.parallelize(mdsList);
+                                after = sparkNet.fitMultiDataSet(dsRDD);
+                                break;
+                            case "paths":
+                                JavaRDD<String> pathRdd = sc.parallelize(paths);
+                                after = sparkNet.fitPathsMultiDataSet(pathRdd);
+                                break;
+                            default:
+                                throw new RuntimeException();
+                        }
+                    } else {
+                        //Fitting from DataSet
                         switch (s) {
                             case "direct":
                             case "export":
@@ -173,6 +205,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
                             default:
                                 throw new RuntimeException();
                         }
+                    }
 
                     INDArray paramsAfter = after.params();
                     System.out.println(Arrays.toString(paramsBefore.get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat()));
@@ -199,6 +232,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
                 lastDup = last.dup();
             }
         }
+    }
 
 
     @Test
@@ -289,7 +323,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
     }
 
 
-    @Test
+    @Test @Ignore
     public void testEpochUpdating() throws Exception {
         //Ensure that epoch counter is incremented properly on the workers
 
@@ -316,7 +350,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest {
 
         ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                 .seed(12345)
-                .updater(new AMSGrad(0.1))
+                .updater(new AMSGrad(0.001))
                 .graphBuilder()
                 .addInputs("in")
                 .layer("out", new OutputLayer.Builder().nIn(784).nOut(10).activation(Activation.SOFTMAX)

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/resources/log4j.properties

@@ -20,12 +20,12 @@ log4j.appender.Console.layout=org.apache.log4j.PatternLayout
 log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n
 
 log4j.appender.org.springframework=DEBUG
-log4j.appender.org.deeplearning4j=DEBUG
+log4j.appender.org.deeplearning4j=INFO
-log4j.appender.org.nd4j=DEBUG
+log4j.appender.org.nd4j=INFO
 
 log4j.logger.org.springframework=INFO
-log4j.logger.org.deeplearning4j=DEBUG
+log4j.logger.org.deeplearning4j=INFO
-log4j.logger.org.nd4j=DEBUG
+log4j.logger.org.nd4j=INFO
 log4j.logger.org.apache.spark=WARN
 
 

deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/resources/logback.xml

@@ -35,7 +35,7 @@
 
     <logger name="org.apache.catalina.core" level="DEBUG" />
     <logger name="org.springframework" level="DEBUG" />
-    <logger name="org.deeplearning4j" level="DEBUG" />
+    <logger name="org.deeplearning4j" level="INFO" />
     <logger name="org.datavec" level="INFO" />
     <logger name="org.nd4j" level="INFO" />
     <logger name="opennlp.uima.util" level="OFF" />

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/pom.xml

@@ -25,10 +25,6 @@
 
     <artifactId>deeplearning4j-ui-components</artifactId>
 
-    <properties>
-        <freemarker.version>2.3.23</freemarker.version>
-    </properties>
-
     <dependencies>
         <dependency>
             <groupId>org.projectlombok</groupId>

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java

@@ -24,6 +24,7 @@ import org.deeplearning4j.ui.components.chart.style.StyleChart;
 import org.deeplearning4j.ui.components.table.ComponentTable;
 import org.deeplearning4j.ui.components.table.style.StyleTable;
 import org.deeplearning4j.ui.standalone.StaticPageUtil;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.awt.*;

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/pom.xml

@@ -60,7 +60,7 @@
         <dependency>
             <groupId>org.freemarker</groupId>
             <artifactId>freemarker</artifactId>
-            <version>2.3.29</version>
+            <version>${freemarker.version}</version>
         </dependency>
 
         <dependency>

deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java

@@ -200,6 +200,7 @@ public class TrainModule implements UIModule {
             }));
             r.add(new Route("/train/:sessionId/info", HttpMethod.GET, (path, rc) -> this.sessionInfoForSession(path.get(0), rc)));
         } else {
+            r.add(new Route("/train", HttpMethod.GET, (path, rc) -> rc.reroute("/train/overview")));
             r.add(new Route("/train/sessions/current", HttpMethod.GET, (path, rc) -> rc.response().end(currentSessionID == null ? "" : currentSessionID)));
             r.add(new Route("/train/sessions/set/:to", HttpMethod.GET, (path, rc) -> this.setSession(path.get(0), rc)));
             r.add(new Route("/train/overview", HttpMethod.GET, (path, rc) -> this.renderFtl("TrainingOverview.html.ftl", rc)));

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SDVariable.java

@@ -1654,29 +1654,6 @@ public class SDVariable implements Serializable {
         return x;
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof SDVariable)) {
-            return false;
-        }
-
-        SDVariable that = (SDVariable) o;
-
-        if (!Objects.equals(varName, that.varName)) {
-            return false;
-        }
-        if (variableType != that.variableType) {
-            return false;
-        }
-        if(sameDiff != that.sameDiff){
-            return false;
-        }
-        return dataType == that.dataType;
-    }
-
     @Override
     public int hashCode() {
         int result = super.hashCode();
@@ -1695,4 +1672,26 @@ public class SDVariable implements Serializable {
         v.sameDiff = sd;
         return v;
     }
+
+    @Override
+    public boolean equals(Object o){
+        if(o == this) return true;
+        if(!(o instanceof SDVariable))
+            return false;
+
+        SDVariable s = (SDVariable)o;
+        if(!varName.equals(s.varName))
+            return false;
+        if(variableType != s.variableType)
+            return false;
+        if(dataType != s.dataType)
+            return false;
+
+        if(variableType == VariableType.VARIABLE || variableType == VariableType.CONSTANT){
+            INDArray a1 = getArr();
+            INDArray a2 = s.getArr();
+            return a1.equals(a2);
+        }
+        return true;
+    }
 }

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiff.java

@@ -1234,13 +1234,14 @@ public class SameDiff extends SDBaseOps {
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (o == null || getClass() != o.getClass())
+            return false;
 
         SameDiff sameDiff = (SameDiff) o;
 
-        if (variables != null ? !variables.equals(sameDiff.variables) : sameDiff.variables != null)
+        boolean eqVars = variables.equals(sameDiff.variables);
-            return false;
+        boolean eqOps = ops.equals(sameDiff.ops);
-        return sameDiffFunctionInstances != null ? sameDiffFunctionInstances.equals(sameDiff.sameDiffFunctionInstances) : sameDiff.sameDiffFunctionInstances == null;
+        return eqVars && eqOps;
     }
 
     /**
@@ -5843,4 +5844,10 @@ public class SameDiff extends SDBaseOps {
 
         return base + "_" + inc;
     }
+
+
+    @Override
+    public String toString(){
+        return "SameDiff(nVars=" + variables.size() + ",nOps=" + ops.size() + ")";
+    }
 }

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java

@@ -16,10 +16,7 @@
 
 package org.nd4j.autodiff.samediff.internal;
 
-import lombok.AllArgsConstructor;
+import lombok.*;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
 import org.nd4j.autodiff.samediff.SDVariable;
 
 import java.util.List;
@@ -28,6 +25,7 @@ import java.util.List;
 @NoArgsConstructor
 @Data   //TODO immutable?
 @Builder
+@EqualsAndHashCode(exclude = {"gradient", "variableIndex"})
 public class Variable {
     protected String name;
     protected SDVariable variable;

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java

@@ -173,9 +173,6 @@ public class EvaluationBinary extends BaseEvaluation<EvaluationBinary> {
 
     @Override
     public void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray, List<? extends Serializable> recordMetaData) {
-        if(recordMetaData != null){
-            throw new UnsupportedOperationException("Evaluation with record metadata not yet implemented for EvaluationBinary");
-        }
         eval(labels, networkPredictions, maskArray);
     }
 

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java

@@ -325,7 +325,7 @@ public class EvaluationCalibration extends BaseEvaluation<EvaluationCalibration>
 
     @Override
     public void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray, List<? extends Serializable> recordMetaData) {
-        throw new UnsupportedOperationException("Not yet implemented");
+        eval(labels, networkPredictions, maskArray);
     }
 
     @Override

nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java

@@ -229,7 +229,7 @@ public class RegressionEvaluation extends BaseEvaluation<RegressionEvaluation> {
 
     @Override
     public void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray, List<? extends Serializable> recordMetaData) {
-        throw new UnsupportedOperationException("Not yet implemented");
+        eval(labels, networkPredictions, maskArray);
     }
 
     @Override

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/autodiff/samediff/SameDiffTests.java

@@ -3556,4 +3556,52 @@ public class SameDiffTests extends BaseNd4jTest {
             assertTrue(msg, msg.contains("\"labels\"") && msg.contains("No array was provided"));
         }
     }
+
+
+    @Test
+    public void testEquals1(){
+
+        SameDiff sd1 = SameDiff.create();
+        SameDiff sd2 = SameDiff.create();
+
+        assertEquals(sd1, sd2);
+
+        SDVariable p1 = sd1.placeHolder("ph", DataType.FLOAT, -1, 10);
+        SDVariable p2 = sd2.placeHolder("ph", DataType.FLOAT, -1, 10);
+
+        assertEquals(sd1, sd2);
+
+        SDVariable w1 = sd1.constant("c1",1.0f);
+        SDVariable w2 = sd2.constant("c1",1.0f);
+
+        assertEquals(sd1, sd2);
+
+        SDVariable a1 = p1.add("add", w1);
+        SDVariable a2 = p2.add("add", w2);
+
+        assertEquals(sd1, sd2);
+
+        SDVariable w1a = sd1.constant("c2", 2.0f);
+        SDVariable w2a = sd2.constant("cX", 2.0f);
+
+        assertNotEquals(sd1, sd2);
+        w2a.rename("c2");
+
+        assertEquals(sd1, sd2);
+
+        sd2.createGradFunction("ph");
+
+        assertEquals(sd1, sd2);
+
+        w2a.getArr().assign(3.0f);
+
+        assertNotEquals(sd1, sd2);
+
+        w1a.getArr().assign(3.0f);
+        assertEquals(sd1, sd2);
+
+        SDVariable s1 = p1.sub("op", w1);
+        SDVariable s2 = p2.add("op", w1);
+        assertNotEquals(sd1, sd2);
+    }
 }

nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/nativ/OpsMappingTests.java

@@ -61,7 +61,7 @@ public class OpsMappingTests extends BaseNd4jTest {
 
     @Override
     public long getTimeoutMilliseconds() {
-        return 90000L;
+        return 180000L;     //Can be slow on some CI machines such as PPC
     }
 
     @Test

nd4j/nd4j-common/src/main/java/org/nd4j/resources/Downloader.java

@@ -95,7 +95,7 @@ public class Downloader {
             }
             // try extracting
             try{
-                ArchiveUtils.unzipFileTo(f.getAbsolutePath(), extractToDir.getAbsolutePath());
+                ArchiveUtils.unzipFileTo(f.getAbsolutePath(), extractToDir.getAbsolutePath(), false);
             } catch (Throwable t){
                 log.warn("Error extracting {} files from file {} - retrying...", name, f.getAbsolutePath(), t);
                 f.delete();

nd4j/nd4j-common/src/main/java/org/nd4j/util/ArchiveUtils.java

@@ -51,6 +51,10 @@ public class ArchiveUtils {
      * @throws IOException
      */
     public static void unzipFileTo(String file, String dest) throws IOException {
+        unzipFileTo(file, dest, true);
+    }
+
+    public static void unzipFileTo(String file, String dest, boolean logFiles) throws IOException {
         File target = new File(file);
         if (!target.exists())
             throw new IllegalArgumentException("Archive doesnt exist");
@@ -93,7 +97,9 @@ public class ArchiveUtils {
 
                     fos.close();
                     ze = zis.getNextEntry();
-                    log.debug("File extracted: " + newFile.getAbsoluteFile());
+                    if(logFiles) {
+                        log.info("File extracted: " + newFile.getAbsoluteFile());
+                    }
                 }
 
                 zis.closeEntry();
@@ -112,7 +118,9 @@ public class ArchiveUtils {
             TarArchiveEntry entry;
             /* Read the tar entries using the getNextEntry method **/
             while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
+                if(logFiles) {
                     log.info("Extracting: " + entry.getName());
+                }
                 /* If the entry is a directory, create the directory. */
 
                 if (entry.isDirectory()) {