Brutex
/
cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

datavec-data-image test fixes 

Signed-off-by: brian <brian@brutex.de>
master
Brian Rosenberger 2022-10-10 23:20:18 +02:00
parent a4bf1c3e62
commit 82dec223ac
25 changed files with 213 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java
View File

@ -27,6 +27,7 @@ import org.datavec.api.split.streams.FileStreamCreatorFunction;
import org.datavec.api.writable.Writable;
import org.nd4j.common.function.Function;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
@ -84,4 +85,54 @@ public abstract class BaseRecordReader implements RecordReader {
public List<List<Writable>> next(int num) {
throw new UnsupportedOperationException();
}
/**
* Closes this resource, relinquishing any underlying resources.
* This method is invoked automatically on objects managed by the
* {@code try}-with-resources statement.
*
* <p>While this interface method is declared to throw {@code
* Exception}, implementers are <em>strongly</em> encouraged to
* declare concrete implementations of the {@code close} method to
* throw more specific exceptions, or to throw no exception at all
* if the close operation cannot fail.
*
* <p> Cases where the close operation may fail require careful
* attention by implementers. It is strongly advised to relinquish
* the underlying resources and to internally <em>mark</em> the
* resource as closed, prior to throwing the exception. The {@code
* close} method is unlikely to be invoked more than once and so
* this ensures that the resources are released in a timely manner.
* Furthermore it reduces problems that could arise when the resource
* wraps, or is wrapped, by another resource.
*
* <p><em>Implementers of this interface are also strongly advised
* to not have the {@code close} method throw {@link
* InterruptedException}.</em>
* <p>
* This exception interacts with a thread's interrupted status,
* and runtime misbehavior is likely to occur if an {@code
* InterruptedException} is {@linkplain Throwable#addSuppressed
* suppressed}.
* <p>
* More generally, if it would cause problems for an
* exception to be suppressed, the {@code AutoCloseable.close}
* method should not throw it.
*
* <p>Note that unlike the {@link Closeable#close close}
* method of {@link Closeable}, this {@code close} method
* is <em>not</em> required to be idempotent. In other words,
* calling this {@code close} method more than once may have some
* visible side effect, unlike {@code Closeable.close} which is
* required to have no effect if called more than once.
* <p>
* However, implementers of this interface are strongly encouraged
* to make their {@code close} methods idempotent.
*
* @throws Exception if this resource cannot be closed
*/
@Override
public void close() throws Exception {
inputSplit.close();
}
}

29
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java
View File

@ -22,11 +22,11 @@ package org.datavec.api.records.reader;
import org.datavec.api.conf.Configurable;
import org.datavec.api.conf.Configuration;
import org.datavec.api.Record;
import org.datavec.api.records.Record;
import org.datavec.api.records.listener.RecordListener;
import org.datavec.api.records.metadata.RecordMetaData;
import org.datavec.api.split.InputSplit;
import org.datavec.api.Writable;
import org.datavec.api.writable.Writable;
import java.io.*;
import java.net.URI;
@ -60,8 +60,7 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException;
/**
* This method returns true, if next(int) signature is supported by this RecordReader
* implementation.
* This method returns true, if next(int) signature is supported by this RecordReader implementation.
*
* @return
*/
@ -83,6 +82,7 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
List<Writable> next();
/**
* Whether there are anymore records
*
@ -99,20 +99,21 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
/**
* Reset record reader iterator
*
* @return
*/
void reset();
/**
* @return True if the record reader can be reset, false otherwise. Note that some record readers
* cannot be reset - for example, if they are backed by a non-resettable input split (such as
* certain types of streams)
* @return True if the record reader can be reset, false otherwise. Note that some record readers cannot be reset -
* for example, if they are backed by a non-resettable input split (such as certain types of streams)
*/
boolean resetSupported();
/**
* Load the record from the given DataInputStream Unlike {@link #next()} the internal state of the
* RecordReader is not modified Implementations of this method should not close the
* DataInputStream
* Load the record from the given DataInputStream
* Unlike {@link #next()} the internal state of the RecordReader is not modified
* Implementations of this method should not close the DataInputStream
*
* @throws IOException if error occurs during reading from the input stream
*/
@ -120,16 +121,16 @@ public interface RecordReader extends AutoCloseable, Serializable, Configurable
/**
* Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such
* as the source of the data
* Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
* of the data
*
* @return next record
*/
Record nextRecord();
/**
* Load a single record from the given {@link RecordMetaData} instance<br> Note: that for data
* that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
* Load a single record from the given {@link RecordMetaData} instance<br>
* Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
* load multiple records at once using {@link #loadFromMetaData(List)}
*
* @param recordMetaData Metadata for the record that we want to load from

5
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java
View File

@ -82,10 +82,11 @@ public class ComposableRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
for (RecordReader reader : readers)
public void close() throws Exception {
for (RecordReader reader : readers) {
reader.close();
}
}
@Override
public void setConf(Configuration conf) {

2
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java
View File

@ -80,7 +80,7 @@ public class ConcatenatingRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
public void close() throws Exception {
for (RecordReader reader : readers)
reader.close();
}

5
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java
View File

@ -68,11 +68,6 @@ public class CollectionRecordReader extends BaseRecordReader {
return records.hasNext();
}
@Override
public void close() throws IOException {
}
@Override
public void setConf(Configuration conf) {

5
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java
View File

@ -72,11 +72,6 @@ public class CollectionSequenceRecordReader extends BaseRecordReader implements
return records.hasNext();
}
@Override
public void close() throws IOException {
}
@Override
public void setConf(Configuration conf) {

18
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java
View File

@ -150,24 +150,6 @@ public class ListStringRecordReader extends BaseRecordReader {
throw new UnsupportedOperationException("Loading from metadata not yet implemented");
}
/**
* Closes this stream and releases any system resources associated
* with it. If the stream is already closed then invoking this
* method has no effect.
* <p>
* <p> As noted in {@link AutoCloseable#close()}, cases where the
* close may fail require careful attention. It is strongly advised
* to relinquish the underlying resources and to internally
* <em>mark</em> the {@code Closeable} as closed, prior to throwing
* the {@code IOException}.
*
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
}
/**
* Set the configuration to be used by this object.
*

2
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java
View File

@ -152,7 +152,7 @@ public class FileBatchRecordReader implements RecordReader {
}
@Override
public void close() throws IOException {
public void close() throws Exception {
recordReader.close();
}

2
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java
View File

@ -172,7 +172,7 @@ public class FileBatchSequenceRecordReader implements SequenceRecordReader {
}
@Override
public void close() throws IOException {
public void close() throws Exception {
recordReader.close();
}

2
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java
View File

@ -258,7 +258,7 @@ public class TransformProcessRecordReader implements RecordReader {
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
public void close() throws Exception {
recordReader.close();
}

2
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java
View File

@ -308,7 +308,7 @@ public class TransformProcessSequenceRecordReader implements SequenceRecordReade
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
public void close() throws Exception {
sequenceRecordReader.close();
}
}

8
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java
View File

@ -20,6 +20,7 @@
package org.datavec.api.split;
import lombok.Getter;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
@ -36,7 +37,7 @@ import java.util.*;
public class FileSplit extends BaseInputSplit {
protected File rootDir;
@Getter protected File rootDir;
// Use for Collections, pass in list of file type strings
protected String[] allowFormat = null;
protected boolean recursive = true;
@ -227,11 +228,6 @@ public class FileSplit extends BaseInputSplit {
}
public File getRootDir() {
return rootDir;
}
private List<File> listFiles(File dir, String[] allowedFormats, boolean recursive) {
Preconditions.checkState(dir.isDirectory(), "Argument is not a directory: %s", dir);
IOFileFilter filter;

6
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java
View File

@ -26,7 +26,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.util.Iterator;
public interface InputSplit {
public interface InputSplit extends AutoCloseable {
/**
@ -134,8 +134,4 @@ public interface InputSplit {
*/
boolean resetSupported();
/**
* Close input/ output streams if any
*/
void close();
}

3
cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java
View File

@ -115,9 +115,8 @@ public class TransformSplit extends BaseInputSplit {
* Close input/ output streams if any
*/
@Override
public void close() {
public void close() throws Exception {
sourceSplit.close();
}
public interface URITransform {

4
cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java
View File

@ -230,8 +230,10 @@ public class CSVSequenceRecordReaderTest extends BaseND4JTest {
}
@Override
public void close() throws Exception {
}
}

5
cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java
View File

@ -46,6 +46,11 @@ public class InputSplitTests extends BaseND4JTest {
@Test
public void testSample() throws URISyntaxException {
BaseInputSplit split = new BaseInputSplit() {
@Override
public void close() throws Exception {
}
{
String[] paths = {"label0/group1_img.tif", "label1/group1_img.jpg", "label2/group1_img.png",
"label3/group1_img.jpeg", "label4/group1_img.bmp", "label5/group1_img.JPEG",

15
cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java → cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/partition/PartitionerTests.java
View File

@ -18,19 +18,16 @@
* *****************************************************************************
*/
package org.datavec.api.split.parittion;
package org.datavec.api.split.partition;
import org.junit.jupiter.api.io.TempDir;
import org.nd4j.common.tests.BaseND4JTest;
import com.google.common.io.Files;
import org.datavec.api.conf.Configuration;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
import org.datavec.api.split.partition.PartitionMetaData;
import org.datavec.api.split.partition.Partitioner;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.OutputStream;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@ -38,9 +35,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class PartitionerTests extends BaseND4JTest {
@Test
public void testRecordsPerFilePartition() {
public void testRecordsPerFilePartition(@TempDir Path tmpDir) {
Partitioner partitioner = new NumberOfRecordsPartitioner();
File tmpDir = Files.createTempDir();
FileSplit fileSplit = new FileSplit(tmpDir);
assertTrue(fileSplit.needsBootstrapForWrite());
fileSplit.bootStrapForWrite();
@ -49,9 +45,8 @@ public class PartitionerTests extends BaseND4JTest {
}
@Test
public void testInputAddFile() throws Exception {
public void testInputAddFile(@TempDir Path tmpDir) throws Exception {
Partitioner partitioner = new NumberOfRecordsPartitioner();
File tmpDir = Files.createTempDir();
FileSplit fileSplit = new FileSplit(tmpDir);
assertTrue(fileSplit.needsBootstrapForWrite());
fileSplit.bootStrapForWrite();

5
cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java
View File

@ -154,11 +154,6 @@ public abstract class BaseAudioRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;

13
cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java
View File

@ -336,8 +336,7 @@ public class NativeImageLoader extends BaseImageLoader {
@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
throw new RuntimeException("Deprecated. Not implemented.");
/*Mat mat = streamToMat(inputStream);
Mat mat = streamToMat(inputStream);
Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
if (image == null || image.empty()) {
PIX pix = pixReadMem(mat.data(), mat.cols());
@ -348,14 +347,20 @@ public class NativeImageLoader extends BaseImageLoader {
pixDestroy(pix);
}
INDArray a = asMatrix(image);
if(!nchw)
a = a.permute(0,2,3,1); //NCHW to NHWC
if(!nchw) a = swapNCHWtoNHWC(a);
Image i = new Image(a, image.channels(), image.rows(), image.cols());
image.deallocate();
return i;
}
/**
* Change channel order from NCHW to NHWC
* @param a input INDArray
* @return swapped INDArray
*/
private INDArray swapNCHWtoNHWC(INDArray a) {
return a.permute(0,2,3,1); //NCHW to NHWC
}
/**

4
cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java
View File

@ -405,8 +405,8 @@ public abstract class BaseImageRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
this.inputSplit.close();
public void close() throws Exception {
inputSplit.close();
}
@Override

2
cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java
View File

@ -80,7 +80,7 @@ public class LoaderTests {
String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin";
String path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
byte[] fullDataExpected = new byte[3073];
FileInputStream inExpected = new FileInputStream(new File(path));
FileInputStream inExpected = new FileInputStream(path);
inExpected.read(fullDataExpected);
byte[] fullDataActual = new byte[3073];

16
cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java
View File

@ -45,6 +45,7 @@ import org.nd4j.common.io.ClassPathResource;
import java.io.File;
import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -54,18 +55,13 @@ import static org.junit.jupiter.api.Assertions.*;
public class TestObjectDetectionRecordReader {
@TempDir
public File testDir;
@Test
public void test() throws Exception {
public void test(@TempDir Path testDir) throws Exception {
for(boolean nchw : new boolean[]{true, false}) {
ImageObjectLabelProvider lp = new TestImageObjectDetectionLabelProvider();
File f = testDir;
new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(f);
String path = new File(f, "000012.jpg").getParent();
new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(testDir);
Path path = testDir.resolve("000012.jpg").getParent();
int h = 32;
int w = 32;
@ -74,7 +70,7 @@ public class TestObjectDetectionRecordReader {
int gH = 10;
//Enforce consistent iteration order for tests
URI[] u = new FileSplit(new File(path)).locations();
URI[] u = new FileSplit(path).locations();
Arrays.sort(u);
RecordReader rr = new ObjectDetectionRecordReader(h, w, c, gH, gW, nchw, lp);
@ -154,7 +150,7 @@ public class TestObjectDetectionRecordReader {
rr.reset();
Record record = rr.nextRecord();
RecordMetaDataImageURI metadata = (RecordMetaDataImageURI) record.getMetaData();
assertEquals(new File(path, "000012.jpg"), new File(metadata.getURI()));
assertEquals( path.resolve( "000012.jpg").toFile(), new File(metadata.getURI()));
assertEquals(3, metadata.getOrigC());
assertEquals((int) origH[0], metadata.getOrigH());
assertEquals((int) origW[0], metadata.getOrigW());

3
cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java
View File

@ -44,9 +44,7 @@ import java.lang.reflect.Array;
import java.util.*;
@Slf4j
@Builder
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
public class DynamicCustomOp extends DifferentialFunction implements CustomOp {
private String opName;
@ -56,7 +54,6 @@ public class DynamicCustomOp extends DifferentialFunction implements CustomOp {
@Builder.Default
protected List<INDArray> outputArguments = new ArrayList<>();
@Builder.Default
protected List<Double> tArguments = new ArrayList<>();

6
cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java
View File

@ -33,6 +33,7 @@ import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
@ -143,7 +144,6 @@ public class ClassPathResource extends AbstractFileResolvingResource {
public void copyDirectory(File destination) throws IOException {
Preconditions.checkState(destination.exists() && destination.isDirectory(), "Destination directory must exist and be a directory: %s", destination);
URL url = this.getUrl();
if (isJarURL(url)) {
@ -180,6 +180,7 @@ public class ClassPathResource extends AbstractFileResolvingResource {
try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(extractTo))){
InputStream is = getInputStream(name, clazz, classLoader);
IOUtils.copy(is, bos);
is.close();
}
}
}
@ -209,6 +210,9 @@ public class ClassPathResource extends AbstractFileResolvingResource {
}
}
public void copyDirectory(Path destination) throws IOException {
copyDirectory(destination.toFile());
}
public boolean exists() {
URL url;
if (this.clazz != null) {

2
cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle
View File

@ -27,4 +27,6 @@ dependencies {
implementation projects.cavisDnn.cavisDnnApi
implementation projects.cavisDatavec.cavisDatavecApi
implementation "commons-io:commons-io"
implementation "com.fasterxml.jackson.core:jackson-annotations"
implementation "com.fasterxml.jackson.core:jackson-core"
}