diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java index 1014dfce4..9fd71d227 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java @@ -27,6 +27,7 @@ import org.datavec.api.split.streams.FileStreamCreatorFunction; import org.datavec.api.writable.Writable; import org.nd4j.common.function.Function; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -84,4 +85,54 @@ public abstract class BaseRecordReader implements RecordReader { public List> next(int num) { throw new UnsupportedOperationException(); } + + /** + * Closes this resource, relinquishing any underlying resources. + * This method is invoked automatically on objects managed by the + * {@code try}-with-resources statement. + * + *

While this interface method is declared to throw {@code + * Exception}, implementers are strongly encouraged to + * declare concrete implementations of the {@code close} method to + * throw more specific exceptions, or to throw no exception at all + * if the close operation cannot fail. + * + *

Cases where the close operation may fail require careful + * attention by implementers. It is strongly advised to relinquish + * the underlying resources and to internally mark the + * resource as closed, prior to throwing the exception. The {@code + * close} method is unlikely to be invoked more than once and so + * this ensures that the resources are released in a timely manner. + * Furthermore it reduces problems that could arise when the resource + * wraps, or is wrapped, by another resource. + * + *

Implementers of this interface are also strongly advised + * to not have the {@code close} method throw {@link + * InterruptedException}. + *

+ * This exception interacts with a thread's interrupted status, + * and runtime misbehavior is likely to occur if an {@code + * InterruptedException} is {@linkplain Throwable#addSuppressed + * suppressed}. + *

+ * More generally, if it would cause problems for an + * exception to be suppressed, the {@code AutoCloseable.close} + * method should not throw it. + * + *

Note that unlike the {@link Closeable#close close} + * method of {@link Closeable}, this {@code close} method + * is not required to be idempotent. In other words, + * calling this {@code close} method more than once may have some + * visible side effect, unlike {@code Closeable.close} which is + * required to have no effect if called more than once. + *

+ * However, implementers of this interface are strongly encouraged + * to make their {@code close} methods idempotent. + * + * @throws Exception if this resource cannot be closed + */ + @Override + public void close() throws Exception { + inputSplit.close(); + } } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java index a124ade3e..14d1da31d 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java @@ -22,11 +22,11 @@ package org.datavec.api.records.reader; import org.datavec.api.conf.Configurable; import org.datavec.api.conf.Configuration; -import org.datavec.api.Record; +import org.datavec.api.records.Record; import org.datavec.api.records.listener.RecordListener; import org.datavec.api.records.metadata.RecordMetaData; import org.datavec.api.split.InputSplit; -import org.datavec.api.Writable; +import org.datavec.api.writable.Writable; import java.io.*; import java.net.URI; @@ -35,130 +35,131 @@ import java.util.List; public interface RecordReader extends AutoCloseable, Serializable, Configurable { - String NAME_SPACE = RecordReader.class.getName(); + String NAME_SPACE = RecordReader.class.getName(); - String APPEND_LABEL = NAME_SPACE + ".appendlabel"; - String LABELS = NAME_SPACE + ".labels"; + String APPEND_LABEL = NAME_SPACE + ".appendlabel"; + String LABELS = NAME_SPACE + ".labels"; - /** - * Called once at initialization. - * - * @param split the split that defines the range of records to read - * @throws IOException - * @throws InterruptedException - */ - void initialize(InputSplit split) throws IOException, InterruptedException; + /** + * Called once at initialization. + * + * @param split the split that defines the range of records to read + * @throws IOException + * @throws InterruptedException + */ + void initialize(InputSplit split) throws IOException, InterruptedException; - /** - * Called once at initialization. - * - * @param conf a configuration for initialization - * @param split the split that defines the range of records to read - * @throws IOException - * @throws InterruptedException - */ - void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException; + /** + * Called once at initialization. + * + * @param conf a configuration for initialization + * @param split the split that defines the range of records to read + * @throws IOException + * @throws InterruptedException + */ + void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException; - /** - * This method returns true, if next(int) signature is supported by this RecordReader - * implementation. - * - * @return - */ - boolean batchesSupported(); + /** + * This method returns true, if next(int) signature is supported by this RecordReader implementation. + * + * @return + */ + boolean batchesSupported(); - /** - * This method will be used, if batchesSupported() returns true. - * - * @param num - * @return - */ - List> next(int num); + /** + * This method will be used, if batchesSupported() returns true. + * + * @param num + * @return + */ + List> next(int num); - /** - * Get the next record - * - * @return - */ - List next(); + /** + * Get the next record + * + * @return + */ + List next(); - /** - * Whether there are anymore records - * - * @return - */ - boolean hasNext(); - /** - * List of label strings - * - * @return - */ - List getLabels(); + /** + * Whether there are anymore records + * + * @return + */ + boolean hasNext(); - /** - * Reset record reader iterator - */ - void reset(); + /** + * List of label strings + * + * @return + */ + List getLabels(); - /** - * @return True if the record reader can be reset, false otherwise. Note that some record readers - * cannot be reset - for example, if they are backed by a non-resettable input split (such as - * certain types of streams) - */ - boolean resetSupported(); + /** + * Reset record reader iterator + * + * @return + */ + void reset(); - /** - * Load the record from the given DataInputStream Unlike {@link #next()} the internal state of the - * RecordReader is not modified Implementations of this method should not close the - * DataInputStream - * - * @throws IOException if error occurs during reading from the input stream - */ - List record(URI uri, DataInputStream dataInputStream) throws IOException; + /** + * @return True if the record reader can be reset, false otherwise. Note that some record readers cannot be reset - + * for example, if they are backed by a non-resettable input split (such as certain types of streams) + */ + boolean resetSupported(); + + /** + * Load the record from the given DataInputStream + * Unlike {@link #next()} the internal state of the RecordReader is not modified + * Implementations of this method should not close the DataInputStream + * + * @throws IOException if error occurs during reading from the input stream + */ + List record(URI uri, DataInputStream dataInputStream) throws IOException; - /** - * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such - * as the source of the data - * - * @return next record - */ - Record nextRecord(); + /** + * Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source + * of the data + * + * @return next record + */ + Record nextRecord(); - /** - * Load a single record from the given {@link RecordMetaData} instance
Note: that for data - * that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to - * load multiple records at once using {@link #loadFromMetaData(List)} - * - * @param recordMetaData Metadata for the record that we want to load from - * @return Single record for the given RecordMetaData instance - * @throws IOException If I/O error occurs during loading - */ - Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException; + /** + * Load a single record from the given {@link RecordMetaData} instance
+ * Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to + * load multiple records at once using {@link #loadFromMetaData(List)} + * + * @param recordMetaData Metadata for the record that we want to load from + * @return Single record for the given RecordMetaData instance + * @throws IOException If I/O error occurs during loading + */ + Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException; - /** - * Load multiple records from the given a list of {@link RecordMetaData} instances
- * - * @param recordMetaDatas Metadata for the records that we want to load from - * @return Multiple records for the given RecordMetaData instances - * @throws IOException If I/O error occurs during loading - */ - List loadFromMetaData(List recordMetaDatas) throws IOException; + /** + * Load multiple records from the given a list of {@link RecordMetaData} instances
+ * + * @param recordMetaDatas Metadata for the records that we want to load from + * @return Multiple records for the given RecordMetaData instances + * @throws IOException If I/O error occurs during loading + */ + List loadFromMetaData(List recordMetaDatas) throws IOException; - /** - * Get the record listeners for this record reader. - */ - List getListeners(); + /** + * Get the record listeners for this record reader. + */ + List getListeners(); - /** - * Set the record listeners for this record reader. - */ - void setListeners(RecordListener... listeners); + /** + * Set the record listeners for this record reader. + */ + void setListeners(RecordListener... listeners); - /** - * Set the record listeners for this record reader. - */ - void setListeners(Collection listeners); + /** + * Set the record listeners for this record reader. + */ + void setListeners(Collection listeners); } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java index 52035e7f1..96693a1a3 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java @@ -82,9 +82,10 @@ public class ComposableRecordReader extends BaseRecordReader { } @Override - public void close() throws IOException { - for (RecordReader reader : readers) - reader.close(); + public void close() throws Exception { + for (RecordReader reader : readers) { + reader.close(); + } } @Override diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java index 14692114e..e01d93ed1 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java @@ -80,7 +80,7 @@ public class ConcatenatingRecordReader extends BaseRecordReader { } @Override - public void close() throws IOException { + public void close() throws Exception { for (RecordReader reader : readers) reader.close(); } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java index 760ea75cb..a8e02e2c4 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java @@ -68,11 +68,6 @@ public class CollectionRecordReader extends BaseRecordReader { return records.hasNext(); } - @Override - public void close() throws IOException { - - } - @Override public void setConf(Configuration conf) { diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java index cf60ba546..c87b541f8 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java @@ -72,11 +72,6 @@ public class CollectionSequenceRecordReader extends BaseRecordReader implements return records.hasNext(); } - @Override - public void close() throws IOException { - - } - @Override public void setConf(Configuration conf) { diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java index 921cef917..7c99ca300 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java @@ -150,24 +150,6 @@ public class ListStringRecordReader extends BaseRecordReader { throw new UnsupportedOperationException("Loading from metadata not yet implemented"); } - /** - * Closes this stream and releases any system resources associated - * with it. If the stream is already closed then invoking this - * method has no effect. - *

- *

As noted in {@link AutoCloseable#close()}, cases where the - * close may fail require careful attention. It is strongly advised - * to relinquish the underlying resources and to internally - * mark the {@code Closeable} as closed, prior to throwing - * the {@code IOException}. - * - * @throws IOException if an I/O error occurs - */ - @Override - public void close() throws IOException { - - } - /** * Set the configuration to be used by this object. * diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java index 219a99870..b827400d6 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java @@ -152,7 +152,7 @@ public class FileBatchRecordReader implements RecordReader { } @Override - public void close() throws IOException { + public void close() throws Exception { recordReader.close(); } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java index 20bdb91b4..133089920 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java @@ -172,7 +172,7 @@ public class FileBatchSequenceRecordReader implements SequenceRecordReader { } @Override - public void close() throws IOException { + public void close() throws Exception { recordReader.close(); } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java index 374b54c45..160b2c134 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java @@ -258,7 +258,7 @@ public class TransformProcessRecordReader implements RecordReader { * @throws IOException if an I/O error occurs */ @Override - public void close() throws IOException { + public void close() throws Exception { recordReader.close(); } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java index 48cd6e937..7023e70b4 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java @@ -308,7 +308,7 @@ public class TransformProcessSequenceRecordReader implements SequenceRecordReade * @throws IOException if an I/O error occurs */ @Override - public void close() throws IOException { + public void close() throws Exception { sequenceRecordReader.close(); } } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java index eb31846a2..97183f346 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java @@ -20,6 +20,7 @@ package org.datavec.api.split; +import lombok.Getter; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.io.filefilter.RegexFileFilter; @@ -36,7 +37,7 @@ import java.util.*; public class FileSplit extends BaseInputSplit { - protected File rootDir; + @Getter protected File rootDir; // Use for Collections, pass in list of file type strings protected String[] allowFormat = null; protected boolean recursive = true; @@ -227,11 +228,6 @@ public class FileSplit extends BaseInputSplit { } - - public File getRootDir() { - return rootDir; - } - private List listFiles(File dir, String[] allowedFormats, boolean recursive) { Preconditions.checkState(dir.isDirectory(), "Argument is not a directory: %s", dir); IOFileFilter filter; diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java index df067ac62..70d6b620b 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java @@ -26,7 +26,7 @@ import java.io.OutputStream; import java.net.URI; import java.util.Iterator; -public interface InputSplit { +public interface InputSplit extends AutoCloseable { /** @@ -134,8 +134,4 @@ public interface InputSplit { */ boolean resetSupported(); - /** - * Close input/ output streams if any - */ - void close(); } diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java index 8c1bda71d..efb3f043a 100644 --- a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java @@ -115,9 +115,8 @@ public class TransformSplit extends BaseInputSplit { * Close input/ output streams if any */ @Override - public void close() { + public void close() throws Exception { sourceSplit.close(); - } public interface URITransform { diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java index 8fdce2165..4032b912c 100644 --- a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java @@ -230,8 +230,10 @@ public class CSVSequenceRecordReaderTest extends BaseND4JTest { } + @Override + public void close() throws Exception { - + } } diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java index f7c413d34..43d274151 100644 --- a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java @@ -46,6 +46,11 @@ public class InputSplitTests extends BaseND4JTest { @Test public void testSample() throws URISyntaxException { BaseInputSplit split = new BaseInputSplit() { + @Override + public void close() throws Exception { + + } + { String[] paths = {"label0/group1_img.tif", "label1/group1_img.jpg", "label2/group1_img.png", "label3/group1_img.jpeg", "label4/group1_img.bmp", "label5/group1_img.JPEG", diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/partition/PartitionerTests.java similarity index 84% rename from cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/partition/PartitionerTests.java index f8e2c99f1..71211c9cd 100644 --- a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/partition/PartitionerTests.java @@ -18,19 +18,16 @@ * ***************************************************************************** */ -package org.datavec.api.split.parittion; +package org.datavec.api.split.partition; +import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; -import com.google.common.io.Files; import org.datavec.api.conf.Configuration; import org.datavec.api.split.FileSplit; -import org.datavec.api.split.partition.NumberOfRecordsPartitioner; -import org.datavec.api.split.partition.PartitionMetaData; -import org.datavec.api.split.partition.Partitioner; import org.junit.jupiter.api.Test; -import java.io.File; import java.io.OutputStream; +import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -38,9 +35,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class PartitionerTests extends BaseND4JTest { @Test - public void testRecordsPerFilePartition() { + public void testRecordsPerFilePartition(@TempDir Path tmpDir) { Partitioner partitioner = new NumberOfRecordsPartitioner(); - File tmpDir = Files.createTempDir(); FileSplit fileSplit = new FileSplit(tmpDir); assertTrue(fileSplit.needsBootstrapForWrite()); fileSplit.bootStrapForWrite(); @@ -49,9 +45,8 @@ public class PartitionerTests extends BaseND4JTest { } @Test - public void testInputAddFile() throws Exception { + public void testInputAddFile(@TempDir Path tmpDir) throws Exception { Partitioner partitioner = new NumberOfRecordsPartitioner(); - File tmpDir = Files.createTempDir(); FileSplit fileSplit = new FileSplit(tmpDir); assertTrue(fileSplit.needsBootstrapForWrite()); fileSplit.bootStrapForWrite(); diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java index 82c9bb1ce..a98b0d1d5 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java @@ -154,11 +154,6 @@ public abstract class BaseAudioRecordReader extends BaseRecordReader { } - @Override - public void close() throws IOException { - - } - @Override public void setConf(Configuration conf) { this.conf = conf; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java index 08a886eb9..bda972a86 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java @@ -336,8 +336,7 @@ public class NativeImageLoader extends BaseImageLoader { @Override public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException { - throw new RuntimeException("Deprecated. Not implemented."); - /*Mat mat = streamToMat(inputStream); + Mat mat = streamToMat(inputStream); Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR); if (image == null || image.empty()) { PIX pix = pixReadMem(mat.data(), mat.cols()); @@ -348,14 +347,20 @@ public class NativeImageLoader extends BaseImageLoader { pixDestroy(pix); } INDArray a = asMatrix(image); - if(!nchw) - a = a.permute(0,2,3,1); //NCHW to NHWC + if(!nchw) a = swapNCHWtoNHWC(a); Image i = new Image(a, image.channels(), image.rows(), image.cols()); image.deallocate(); return i; + } - */ + /** + * Change channel order from NCHW to NHWC + * @param a input INDArray + * @return swapped INDArray + */ + private INDArray swapNCHWtoNHWC(INDArray a) { + return a.permute(0,2,3,1); //NCHW to NHWC } /** diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java index f2644b3ac..48502f95d 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java @@ -405,8 +405,8 @@ public abstract class BaseImageRecordReader extends BaseRecordReader { } @Override - public void close() throws IOException { - this.inputSplit.close(); + public void close() throws Exception { + inputSplit.close(); } @Override diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java index b9b4cf3a2..2ca05dd55 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java @@ -80,7 +80,7 @@ public class LoaderTests { String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin"; String path = FilenameUtils.concat(System.getProperty("user.home"), subDir); byte[] fullDataExpected = new byte[3073]; - FileInputStream inExpected = new FileInputStream(new File(path)); + FileInputStream inExpected = new FileInputStream(path); inExpected.read(fullDataExpected); byte[] fullDataActual = new byte[3073]; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java index 81d667f78..483be1090 100644 --- a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java @@ -45,6 +45,7 @@ import org.nd4j.common.io.ClassPathResource; import java.io.File; import java.net.URI; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -54,18 +55,13 @@ import static org.junit.jupiter.api.Assertions.*; public class TestObjectDetectionRecordReader { - @TempDir - public File testDir; - @Test - public void test() throws Exception { + public void test(@TempDir Path testDir) throws Exception { for(boolean nchw : new boolean[]{true, false}) { ImageObjectLabelProvider lp = new TestImageObjectDetectionLabelProvider(); - File f = testDir; - new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(f); - - String path = new File(f, "000012.jpg").getParent(); + new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(testDir); + Path path = testDir.resolve("000012.jpg").getParent(); int h = 32; int w = 32; @@ -74,7 +70,7 @@ public class TestObjectDetectionRecordReader { int gH = 10; //Enforce consistent iteration order for tests - URI[] u = new FileSplit(new File(path)).locations(); + URI[] u = new FileSplit(path).locations(); Arrays.sort(u); RecordReader rr = new ObjectDetectionRecordReader(h, w, c, gH, gW, nchw, lp); @@ -154,7 +150,7 @@ public class TestObjectDetectionRecordReader { rr.reset(); Record record = rr.nextRecord(); RecordMetaDataImageURI metadata = (RecordMetaDataImageURI) record.getMetaData(); - assertEquals(new File(path, "000012.jpg"), new File(metadata.getURI())); + assertEquals( path.resolve( "000012.jpg").toFile(), new File(metadata.getURI())); assertEquals(3, metadata.getOrigC()); assertEquals((int) origH[0], metadata.getOrigH()); assertEquals((int) origW[0], metadata.getOrigW()); diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java index 14b736898..304f40b99 100644 --- a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java @@ -44,9 +44,7 @@ import java.lang.reflect.Array; import java.util.*; @Slf4j -@Builder @AllArgsConstructor -@EqualsAndHashCode(callSuper = true) public class DynamicCustomOp extends DifferentialFunction implements CustomOp { private String opName; @@ -56,7 +54,6 @@ public class DynamicCustomOp extends DifferentialFunction implements CustomOp { @Builder.Default protected List outputArguments = new ArrayList<>(); - @Builder.Default protected List tArguments = new ArrayList<>(); diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java index 03fa53767..7c5687ef9 100644 --- a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java @@ -33,6 +33,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.util.Enumeration; import java.util.zip.ZipEntry; @@ -143,7 +144,6 @@ public class ClassPathResource extends AbstractFileResolvingResource { public void copyDirectory(File destination) throws IOException { Preconditions.checkState(destination.exists() && destination.isDirectory(), "Destination directory must exist and be a directory: %s", destination); - URL url = this.getUrl(); if (isJarURL(url)) { @@ -180,6 +180,7 @@ public class ClassPathResource extends AbstractFileResolvingResource { try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(extractTo))){ InputStream is = getInputStream(name, clazz, classLoader); IOUtils.copy(is, bos); + is.close(); } } } @@ -209,6 +210,9 @@ public class ClassPathResource extends AbstractFileResolvingResource { } } + public void copyDirectory(Path destination) throws IOException { + copyDirectory(destination.toFile()); + } public boolean exists() { URL url; if (this.clazz != null) { diff --git a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle index 6eabedb4f..814edc3b6 100644 --- a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle @@ -27,4 +27,6 @@ dependencies { implementation projects.cavisDnn.cavisDnnApi implementation projects.cavisDatavec.cavisDatavecApi implementation "commons-io:commons-io" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-core" } \ No newline at end of file