datavec-data-image test fixes

Signed-off-by: brian <brian@brutex.de>

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java

@@ -27,6 +27,7 @@ import org.datavec.api.split.streams.FileStreamCreatorFunction;
 import org.datavec.api.writable.Writable;
 import org.nd4j.common.function.Function;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
@@ -84,4 +85,54 @@ public abstract class BaseRecordReader implements RecordReader {
     public List<List<Writable>> next(int num) {
         throw new UnsupportedOperationException();
     }
+
+    /**
+     * Closes this resource, relinquishing any underlying resources.
+     * This method is invoked automatically on objects managed by the
+     * {@code try}-with-resources statement.
+     *
+     * <p>While this interface method is declared to throw {@code
+     * Exception}, implementers are <em>strongly</em> encouraged to
+     * declare concrete implementations of the {@code close} method to
+     * throw more specific exceptions, or to throw no exception at all
+     * if the close operation cannot fail.
+     *
+     * <p> Cases where the close operation may fail require careful
+     * attention by implementers. It is strongly advised to relinquish
+     * the underlying resources and to internally <em>mark</em> the
+     * resource as closed, prior to throwing the exception. The {@code
+     * close} method is unlikely to be invoked more than once and so
+     * this ensures that the resources are released in a timely manner.
+     * Furthermore it reduces problems that could arise when the resource
+     * wraps, or is wrapped, by another resource.
+     *
+     * <p><em>Implementers of this interface are also strongly advised
+     * to not have the {@code close} method throw {@link
+     * InterruptedException}.</em>
+     * <p>
+     * This exception interacts with a thread's interrupted status,
+     * and runtime misbehavior is likely to occur if an {@code
+     * InterruptedException} is {@linkplain Throwable#addSuppressed
+     * suppressed}.
+     * <p>
+     * More generally, if it would cause problems for an
+     * exception to be suppressed, the {@code AutoCloseable.close}
+     * method should not throw it.
+     *
+     * <p>Note that unlike the {@link Closeable#close close}
+     * method of {@link Closeable}, this {@code close} method
+     * is <em>not</em> required to be idempotent.  In other words,
+     * calling this {@code close} method more than once may have some
+     * visible side effect, unlike {@code Closeable.close} which is
+     * required to have no effect if called more than once.
+     * <p>
+     * However, implementers of this interface are strongly encouraged
+     * to make their {@code close} methods idempotent.
+     *
+     * @throws Exception if this resource cannot be closed
+     */
+    @Override
+    public void close() throws Exception {
+        inputSplit.close();
+    }
 }

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java

@@ -22,11 +22,11 @@ package org.datavec.api.records.reader;
 
 import org.datavec.api.conf.Configurable;
 import org.datavec.api.conf.Configuration;
-import org.datavec.api.Record;
+import org.datavec.api.records.Record;
 import org.datavec.api.records.listener.RecordListener;
 import org.datavec.api.records.metadata.RecordMetaData;
 import org.datavec.api.split.InputSplit;
-import org.datavec.api.Writable;
+import org.datavec.api.writable.Writable;
 
 import java.io.*;
 import java.net.URI;
@@ -60,8 +60,7 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
     void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException;
 
     /**
-   * This method returns true, if next(int) signature is supported by this RecordReader
+     * This method returns true, if next(int) signature is supported by this RecordReader implementation.
-   * implementation.
      *
      * @return
      */
@@ -83,6 +82,7 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
     List<Writable> next();
 
 
+
     /**
      * Whether there are anymore records
      *
@@ -99,20 +99,21 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
 
     /**
      * Reset record reader iterator
+     *
+     * @return
      */
     void reset();
 
     /**
-   * @return True if the record reader can be reset, false otherwise. Note that some record readers
+     * @return True if the record reader can be reset, false otherwise. Note that some record readers cannot be reset -
-   * cannot be reset - for example, if they are backed by a non-resettable input split (such as
+     * for example, if they are backed by a non-resettable input split (such as certain types of streams)
-   * certain types of streams)
      */
     boolean resetSupported();
 
     /**
-   * Load the record from the given DataInputStream Unlike {@link #next()} the internal state of the
+     * Load the record from the given DataInputStream
-   * RecordReader is not modified Implementations of this method should not close the
+     * Unlike {@link #next()} the internal state of the RecordReader is not modified
-   * DataInputStream
+     * Implementations of this method should not close the DataInputStream
      *
      * @throws IOException if error occurs during reading from the input stream
      */
@@ -120,16 +121,16 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
 
 
     /**
-   * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such
+     * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
-   * as the source of the data
+     * of the data
      *
      * @return next record
      */
     Record nextRecord();
 
     /**
-   * Load a single record from the given {@link RecordMetaData} instance<br> Note: that for data
+     * Load a single record from the given {@link RecordMetaData} instance<br>
-   * that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
+     * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
      * load multiple records at once using {@link #loadFromMetaData(List)}
      *
      * @param recordMetaData Metadata for the record that we want to load from

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java

@@ -82,10 +82,11 @@ public class ComposableRecordReader extends BaseRecordReader {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
-        for (RecordReader reader : readers)
+        for (RecordReader reader : readers) {
                 reader.close();
         }
+    }
 
     @Override
     public void setConf(Configuration conf) {

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java

@@ -80,7 +80,7 @@ public class ConcatenatingRecordReader extends BaseRecordReader {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
         for (RecordReader reader : readers)
             reader.close();
     }

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java

@@ -68,11 +68,6 @@ public class CollectionRecordReader extends BaseRecordReader {
         return records.hasNext();
     }
 
-    @Override
-    public void close() throws IOException {
-
-    }
-
     @Override
     public void setConf(Configuration conf) {
 

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java

@@ -72,11 +72,6 @@ public class CollectionSequenceRecordReader extends BaseRecordReader implements
         return records.hasNext();
     }
 
-    @Override
-    public void close() throws IOException {
-
-    }
-
     @Override
     public void setConf(Configuration conf) {
 

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java

@@ -150,24 +150,6 @@ public class ListStringRecordReader extends BaseRecordReader {
         throw new UnsupportedOperationException("Loading from metadata not yet implemented");
     }
 
-    /**
-     * Closes this stream and releases any system resources associated
-     * with it. If the stream is already closed then invoking this
-     * method has no effect.
-     * <p>
-     * <p> As noted in {@link AutoCloseable#close()}, cases where the
-     * close may fail require careful attention. It is strongly advised
-     * to relinquish the underlying resources and to internally
-     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
-     * the {@code IOException}.
-     *
-     * @throws IOException if an I/O error occurs
-     */
-    @Override
-    public void close() throws IOException {
-
-    }
-
     /**
      * Set the configuration to be used by this object.
      *

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java

@@ -152,7 +152,7 @@ public class FileBatchRecordReader implements RecordReader {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
         recordReader.close();
     }
 

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java

@@ -172,7 +172,7 @@ public class FileBatchSequenceRecordReader implements SequenceRecordReader {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
         recordReader.close();
     }
 

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java

@@ -258,7 +258,7 @@ public class TransformProcessRecordReader implements RecordReader {
      * @throws IOException if an I/O error occurs
      */
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
         recordReader.close();
     }
 

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java

@@ -308,7 +308,7 @@ public class TransformProcessSequenceRecordReader implements SequenceRecordReade
      * @throws IOException if an I/O error occurs
      */
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
         sequenceRecordReader.close();
     }
 }

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java

@@ -20,6 +20,7 @@
 
 package org.datavec.api.split;
 
+import lombok.Getter;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.commons.io.filefilter.RegexFileFilter;
@@ -36,7 +37,7 @@ import java.util.*;
 
 public class FileSplit extends BaseInputSplit {
 
-    protected File rootDir;
+    @Getter protected File rootDir;
     // Use for Collections, pass in list of file type strings
     protected String[] allowFormat = null;
     protected boolean recursive = true;
@@ -227,11 +228,6 @@ public class FileSplit extends BaseInputSplit {
 
     }
 
-
-    public File getRootDir() {
-        return rootDir;
-    }
-
     private List<File> listFiles(File dir, String[] allowedFormats, boolean recursive) {
         Preconditions.checkState(dir.isDirectory(), "Argument is not a directory: %s", dir);
         IOFileFilter filter;

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java

@@ -26,7 +26,7 @@ import java.io.OutputStream;
 import java.net.URI;
 import java.util.Iterator;
 
-public interface InputSplit  {
+public interface InputSplit extends AutoCloseable {
 
 
     /**
@@ -134,8 +134,4 @@ public interface InputSplit  {
      */
     boolean resetSupported();
 
-    /**
-     * Close input/ output streams if any
-     */
-    void close();
 }

cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java

@@ -115,9 +115,8 @@ public class TransformSplit extends BaseInputSplit {
      * Close input/ output streams if any
      */
     @Override
-    public void close() {
+    public void close() throws Exception {
         sourceSplit.close();
-
     }
 
     public interface URITransform {

cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java

@@ -230,8 +230,10 @@ public class CSVSequenceRecordReaderTest  extends BaseND4JTest {
         }
 
 
+        @Override
+        public void close() throws Exception {
 
-
+        }
     }
 
 

cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java

@@ -46,6 +46,11 @@ public class InputSplitTests extends BaseND4JTest {
     @Test
     public void testSample() throws URISyntaxException {
         BaseInputSplit split = new BaseInputSplit() {
+            @Override
+            public void close() throws Exception {
+
+            }
+
             {
                 String[] paths = {"label0/group1_img.tif", "label1/group1_img.jpg", "label2/group1_img.png",
                                 "label3/group1_img.jpeg", "label4/group1_img.bmp", "label5/group1_img.JPEG",

cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/partition/PartitionerTests.java

@@ -18,19 +18,16 @@
  *  *****************************************************************************
  */
 
-package org.datavec.api.split.parittion;
+package org.datavec.api.split.partition;
 
+import org.junit.jupiter.api.io.TempDir;
 import org.nd4j.common.tests.BaseND4JTest;
-import com.google.common.io.Files;
 import org.datavec.api.conf.Configuration;
 import org.datavec.api.split.FileSplit;
-import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
-import org.datavec.api.split.partition.PartitionMetaData;
-import org.datavec.api.split.partition.Partitioner;
 import org.junit.jupiter.api.Test;
 
-import java.io.File;
 import java.io.OutputStream;
+import java.nio.file.Path;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -38,9 +35,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class PartitionerTests extends BaseND4JTest {
     @Test
-    public void testRecordsPerFilePartition() {
+    public void testRecordsPerFilePartition(@TempDir Path tmpDir) {
         Partitioner partitioner = new NumberOfRecordsPartitioner();
-        File tmpDir = Files.createTempDir();
         FileSplit fileSplit = new FileSplit(tmpDir);
         assertTrue(fileSplit.needsBootstrapForWrite());
         fileSplit.bootStrapForWrite();
@@ -49,9 +45,8 @@ public class PartitionerTests extends BaseND4JTest {
     }
 
     @Test
-    public void testInputAddFile() throws Exception {
+    public void testInputAddFile(@TempDir Path tmpDir) throws Exception {
         Partitioner partitioner = new NumberOfRecordsPartitioner();
-        File tmpDir = Files.createTempDir();
         FileSplit fileSplit = new FileSplit(tmpDir);
         assertTrue(fileSplit.needsBootstrapForWrite());
         fileSplit.bootStrapForWrite();

cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java

@@ -154,11 +154,6 @@ public abstract class BaseAudioRecordReader extends BaseRecordReader {
     }
 
 
-    @Override
-    public void close() throws IOException {
-
-    }
-
     @Override
     public void setConf(Configuration conf) {
         this.conf = conf;

cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java

@@ -336,8 +336,7 @@ public class NativeImageLoader extends BaseImageLoader {
 
     @Override
     public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
-        throw new RuntimeException("Deprecated. Not implemented.");
+        Mat mat = streamToMat(inputStream);
-        /*Mat mat = streamToMat(inputStream);
         Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
         if (image == null || image.empty()) {
             PIX pix = pixReadMem(mat.data(), mat.cols());
@@ -348,14 +347,20 @@ public class NativeImageLoader extends BaseImageLoader {
             pixDestroy(pix);
         }
         INDArray a = asMatrix(image);
-        if(!nchw)
+        if(!nchw) a = swapNCHWtoNHWC(a);
-            a = a.permute(0,2,3,1);     //NCHW to NHWC
         Image i = new Image(a, image.channels(), image.rows(), image.cols());
 
         image.deallocate();
         return i;
+    }
 
+    /**
+     * Change channel order from NCHW to NHWC
+     * @param a input INDArray
+     * @return swapped INDArray
      */
+    private INDArray swapNCHWtoNHWC(INDArray a) {
+        return a.permute(0,2,3,1);     //NCHW to NHWC
     }
 
     /**

cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java

@@ -405,8 +405,8 @@ public abstract class BaseImageRecordReader extends BaseRecordReader {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() throws Exception {
-        this.inputSplit.close();
+        inputSplit.close();
     }
 
     @Override

cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java

@@ -80,7 +80,7 @@ public class LoaderTests {
         String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin";
         String path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
         byte[] fullDataExpected = new byte[3073];
-        FileInputStream inExpected = new FileInputStream(new File(path));
+        FileInputStream inExpected = new FileInputStream(path);
         inExpected.read(fullDataExpected);
 
         byte[] fullDataActual = new byte[3073];

cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java

@@ -45,6 +45,7 @@ import org.nd4j.common.io.ClassPathResource;
 
 import java.io.File;
 import java.net.URI;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -54,18 +55,13 @@ import static org.junit.jupiter.api.Assertions.*;
 
 public class TestObjectDetectionRecordReader {
 
-    @TempDir
-    public File testDir;
-
     @Test
-    public void test() throws Exception {
+    public void test(@TempDir Path testDir) throws Exception {
         for(boolean nchw : new boolean[]{true, false}) {
             ImageObjectLabelProvider lp = new TestImageObjectDetectionLabelProvider();
 
-            File f = testDir;
+            new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(testDir);
-            new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(f);
+            Path path = testDir.resolve("000012.jpg").getParent();
-
-            String path = new File(f, "000012.jpg").getParent();
 
             int h = 32;
             int w = 32;
@@ -74,7 +70,7 @@ public class TestObjectDetectionRecordReader {
             int gH = 10;
 
             //Enforce consistent iteration order for tests
-            URI[] u = new FileSplit(new File(path)).locations();
+            URI[] u = new FileSplit(path).locations();
             Arrays.sort(u);
 
             RecordReader rr = new ObjectDetectionRecordReader(h, w, c, gH, gW, nchw, lp);
@@ -154,7 +150,7 @@ public class TestObjectDetectionRecordReader {
             rr.reset();
             Record record = rr.nextRecord();
             RecordMetaDataImageURI metadata = (RecordMetaDataImageURI) record.getMetaData();
-            assertEquals(new File(path, "000012.jpg"), new File(metadata.getURI()));
+            assertEquals( path.resolve( "000012.jpg").toFile(), new File(metadata.getURI()));
             assertEquals(3, metadata.getOrigC());
             assertEquals((int) origH[0], metadata.getOrigH());
             assertEquals((int) origW[0], metadata.getOrigW());

cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java

@@ -44,9 +44,7 @@ import java.lang.reflect.Array;
 import java.util.*;
 
 @Slf4j
-@Builder
 @AllArgsConstructor
-@EqualsAndHashCode(callSuper = true)
 public class DynamicCustomOp extends DifferentialFunction implements CustomOp {
 
     private String opName;
@@ -56,7 +54,6 @@ public class DynamicCustomOp extends DifferentialFunction implements CustomOp {
     @Builder.Default
     protected List<INDArray> outputArguments = new ArrayList<>();
 
-
     @Builder.Default
     protected List<Double> tArguments = new ArrayList<>();
 

cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java

@@ -33,6 +33,7 @@ import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.attribute.FileAttribute;
 import java.util.Enumeration;
 import java.util.zip.ZipEntry;
@@ -143,7 +144,6 @@ public class ClassPathResource extends AbstractFileResolvingResource {
     public void copyDirectory(File destination) throws IOException {
         Preconditions.checkState(destination.exists() && destination.isDirectory(), "Destination directory must exist and be a directory: %s", destination);
 
-
         URL url = this.getUrl();
 
         if (isJarURL(url)) {
@@ -180,6 +180,7 @@ public class ClassPathResource extends AbstractFileResolvingResource {
                             try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(extractTo))){
                                 InputStream is = getInputStream(name, clazz, classLoader);
                                 IOUtils.copy(is, bos);
+                                is.close();
                             }
                         }
                     }
@@ -209,6 +210,9 @@ public class ClassPathResource extends AbstractFileResolvingResource {
         }
     }
 
+    public void copyDirectory(Path destination) throws IOException {
+        copyDirectory(destination.toFile());
+    }
     public boolean exists() {
         URL url;
         if (this.clazz != null) {

cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle

@@ -27,4 +27,6 @@ dependencies {
     implementation projects.cavisDnn.cavisDnnApi
     implementation projects.cavisDatavec.cavisDatavecApi
     implementation "commons-io:commons-io"
+    implementation "com.fasterxml.jackson.core:jackson-annotations"
+    implementation "com.fasterxml.jackson.core:jackson-core"
 }