datavec-data-image test fixes

Signed-off-by: brian <brian@brutex.de>
master
Brian Rosenberger 2022-10-10 23:20:18 +02:00
parent a4bf1c3e62
commit 82dec223ac
25 changed files with 213 additions and 196 deletions

View File

@ -27,6 +27,7 @@ import org.datavec.api.split.streams.FileStreamCreatorFunction;
import org.datavec.api.writable.Writable;
import org.nd4j.common.function.Function;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
@ -84,4 +85,54 @@ public abstract class BaseRecordReader implements RecordReader {
public List<List<Writable>> next(int num) {
throw new UnsupportedOperationException();
}
/**
* Closes this resource, relinquishing any underlying resources.
* This method is invoked automatically on objects managed by the
* {@code try}-with-resources statement.
*
* <p>While this interface method is declared to throw {@code
* Exception}, implementers are <em>strongly</em> encouraged to
* declare concrete implementations of the {@code close} method to
* throw more specific exceptions, or to throw no exception at all
* if the close operation cannot fail.
*
* <p> Cases where the close operation may fail require careful
* attention by implementers. It is strongly advised to relinquish
* the underlying resources and to internally <em>mark</em> the
* resource as closed, prior to throwing the exception. The {@code
* close} method is unlikely to be invoked more than once and so
* this ensures that the resources are released in a timely manner.
* Furthermore it reduces problems that could arise when the resource
* wraps, or is wrapped, by another resource.
*
* <p><em>Implementers of this interface are also strongly advised
* to not have the {@code close} method throw {@link
* InterruptedException}.</em>
* <p>
* This exception interacts with a thread's interrupted status,
* and runtime misbehavior is likely to occur if an {@code
* InterruptedException} is {@linkplain Throwable#addSuppressed
* suppressed}.
* <p>
* More generally, if it would cause problems for an
* exception to be suppressed, the {@code AutoCloseable.close}
* method should not throw it.
*
* <p>Note that unlike the {@link Closeable#close close}
* method of {@link Closeable}, this {@code close} method
* is <em>not</em> required to be idempotent. In other words,
* calling this {@code close} method more than once may have some
* visible side effect, unlike {@code Closeable.close} which is
* required to have no effect if called more than once.
* <p>
* However, implementers of this interface are strongly encouraged
* to make their {@code close} methods idempotent.
*
* @throws Exception if this resource cannot be closed
*/
@Override
public void close() throws Exception {
inputSplit.close();
}
}

View File

@ -22,11 +22,11 @@ package org.datavec.api.records.reader;
import org.datavec.api.conf.Configurable;
import org.datavec.api.conf.Configuration;
import org.datavec.api.Record;
import org.datavec.api.records.Record;
import org.datavec.api.records.listener.RecordListener;
import org.datavec.api.records.metadata.RecordMetaData;
import org.datavec.api.split.InputSplit;
import org.datavec.api.Writable;
import org.datavec.api.writable.Writable;
import java.io.*;
import java.net.URI;
@ -35,130 +35,131 @@ import java.util.List;
public interface RecordReader extends AutoCloseable, Serializable, Configurable {
String NAME_SPACE = RecordReader.class.getName();
String NAME_SPACE = RecordReader.class.getName();
String APPEND_LABEL = NAME_SPACE + ".appendlabel";
String LABELS = NAME_SPACE + ".labels";
String APPEND_LABEL = NAME_SPACE + ".appendlabel";
String LABELS = NAME_SPACE + ".labels";
/**
* Called once at initialization.
*
* @param split the split that defines the range of records to read
* @throws IOException
* @throws InterruptedException
*/
void initialize(InputSplit split) throws IOException, InterruptedException;
/**
* Called once at initialization.
*
* @param split the split that defines the range of records to read
* @throws IOException
* @throws InterruptedException
*/
void initialize(InputSplit split) throws IOException, InterruptedException;
/**
* Called once at initialization.
*
* @param conf a configuration for initialization
* @param split the split that defines the range of records to read
* @throws IOException
* @throws InterruptedException
*/
void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException;
/**
* Called once at initialization.
*
* @param conf a configuration for initialization
* @param split the split that defines the range of records to read
* @throws IOException
* @throws InterruptedException
*/
void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException;
/**
* This method returns true, if next(int) signature is supported by this RecordReader
* implementation.
*
* @return
*/
boolean batchesSupported();
/**
* This method returns true, if next(int) signature is supported by this RecordReader implementation.
*
* @return
*/
boolean batchesSupported();
/**
* This method will be used, if batchesSupported() returns true.
*
* @param num
* @return
*/
List<List<Writable>> next(int num);
/**
* This method will be used, if batchesSupported() returns true.
*
* @param num
* @return
*/
List<List<Writable>> next(int num);
/**
* Get the next record
*
* @return
*/
List<Writable> next();
/**
* Get the next record
*
* @return
*/
List<Writable> next();
/**
* Whether there are anymore records
*
* @return
*/
boolean hasNext();
/**
* List of label strings
*
* @return
*/
List<String> getLabels();
/**
* Whether there are anymore records
*
* @return
*/
boolean hasNext();
/**
* Reset record reader iterator
*/
void reset();
/**
* List of label strings
*
* @return
*/
List<String> getLabels();
/**
* @return True if the record reader can be reset, false otherwise. Note that some record readers
* cannot be reset - for example, if they are backed by a non-resettable input split (such as
* certain types of streams)
*/
boolean resetSupported();
/**
* Reset record reader iterator
*
* @return
*/
void reset();
/**
* Load the record from the given DataInputStream Unlike {@link #next()} the internal state of the
* RecordReader is not modified Implementations of this method should not close the
* DataInputStream
*
* @throws IOException if error occurs during reading from the input stream
*/
List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException;
/**
* @return True if the record reader can be reset, false otherwise. Note that some record readers cannot be reset -
* for example, if they are backed by a non-resettable input split (such as certain types of streams)
*/
boolean resetSupported();
/**
* Load the record from the given DataInputStream
* Unlike {@link #next()} the internal state of the RecordReader is not modified
* Implementations of this method should not close the DataInputStream
*
* @throws IOException if error occurs during reading from the input stream
*/
List<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException;
/**
* Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such
* as the source of the data
*
* @return next record
*/
Record nextRecord();
/**
* Similar to {@link #next()}, but returns a {@link Record} object, that may include metadata such as the source
* of the data
*
* @return next record
*/
Record nextRecord();
/**
* Load a single record from the given {@link RecordMetaData} instance<br> Note: that for data
* that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
* load multiple records at once using {@link #loadFromMetaData(List)}
*
* @param recordMetaData Metadata for the record that we want to load from
* @return Single record for the given RecordMetaData instance
* @throws IOException If I/O error occurs during loading
*/
Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException;
/**
* Load a single record from the given {@link RecordMetaData} instance<br>
* Note: that for data that isn't splittable (i.e., text data that needs to be scanned/split), it is more efficient to
* load multiple records at once using {@link #loadFromMetaData(List)}
*
* @param recordMetaData Metadata for the record that we want to load from
* @return Single record for the given RecordMetaData instance
* @throws IOException If I/O error occurs during loading
*/
Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException;
/**
* Load multiple records from the given a list of {@link RecordMetaData} instances<br>
*
* @param recordMetaDatas Metadata for the records that we want to load from
* @return Multiple records for the given RecordMetaData instances
* @throws IOException If I/O error occurs during loading
*/
List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException;
/**
* Load multiple records from the given a list of {@link RecordMetaData} instances<br>
*
* @param recordMetaDatas Metadata for the records that we want to load from
* @return Multiple records for the given RecordMetaData instances
* @throws IOException If I/O error occurs during loading
*/
List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException;
/**
* Get the record listeners for this record reader.
*/
List<RecordListener> getListeners();
/**
* Get the record listeners for this record reader.
*/
List<RecordListener> getListeners();
/**
* Set the record listeners for this record reader.
*/
void setListeners(RecordListener... listeners);
/**
* Set the record listeners for this record reader.
*/
void setListeners(RecordListener... listeners);
/**
* Set the record listeners for this record reader.
*/
void setListeners(Collection<RecordListener> listeners);
/**
* Set the record listeners for this record reader.
*/
void setListeners(Collection<RecordListener> listeners);
}

View File

@ -82,9 +82,10 @@ public class ComposableRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
for (RecordReader reader : readers)
reader.close();
public void close() throws Exception {
for (RecordReader reader : readers) {
reader.close();
}
}
@Override

View File

@ -80,7 +80,7 @@ public class ConcatenatingRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
public void close() throws Exception {
for (RecordReader reader : readers)
reader.close();
}

View File

@ -68,11 +68,6 @@ public class CollectionRecordReader extends BaseRecordReader {
return records.hasNext();
}
@Override
public void close() throws IOException {
}
@Override
public void setConf(Configuration conf) {

View File

@ -72,11 +72,6 @@ public class CollectionSequenceRecordReader extends BaseRecordReader implements
return records.hasNext();
}
@Override
public void close() throws IOException {
}
@Override
public void setConf(Configuration conf) {

View File

@ -150,24 +150,6 @@ public class ListStringRecordReader extends BaseRecordReader {
throw new UnsupportedOperationException("Loading from metadata not yet implemented");
}
/**
* Closes this stream and releases any system resources associated
* with it. If the stream is already closed then invoking this
* method has no effect.
* <p>
* <p> As noted in {@link AutoCloseable#close()}, cases where the
* close may fail require careful attention. It is strongly advised
* to relinquish the underlying resources and to internally
* <em>mark</em> the {@code Closeable} as closed, prior to throwing
* the {@code IOException}.
*
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
}
/**
* Set the configuration to be used by this object.
*

View File

@ -152,7 +152,7 @@ public class FileBatchRecordReader implements RecordReader {
}
@Override
public void close() throws IOException {
public void close() throws Exception {
recordReader.close();
}

View File

@ -172,7 +172,7 @@ public class FileBatchSequenceRecordReader implements SequenceRecordReader {
}
@Override
public void close() throws IOException {
public void close() throws Exception {
recordReader.close();
}

View File

@ -258,7 +258,7 @@ public class TransformProcessRecordReader implements RecordReader {
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
public void close() throws Exception {
recordReader.close();
}

View File

@ -308,7 +308,7 @@ public class TransformProcessSequenceRecordReader implements SequenceRecordReade
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
public void close() throws Exception {
sequenceRecordReader.close();
}
}

View File

@ -20,6 +20,7 @@
package org.datavec.api.split;
import lombok.Getter;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
@ -36,7 +37,7 @@ import java.util.*;
public class FileSplit extends BaseInputSplit {
protected File rootDir;
@Getter protected File rootDir;
// Use for Collections, pass in list of file type strings
protected String[] allowFormat = null;
protected boolean recursive = true;
@ -227,11 +228,6 @@ public class FileSplit extends BaseInputSplit {
}
public File getRootDir() {
return rootDir;
}
private List<File> listFiles(File dir, String[] allowedFormats, boolean recursive) {
Preconditions.checkState(dir.isDirectory(), "Argument is not a directory: %s", dir);
IOFileFilter filter;

View File

@ -26,7 +26,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.util.Iterator;
public interface InputSplit {
public interface InputSplit extends AutoCloseable {
/**
@ -134,8 +134,4 @@ public interface InputSplit {
*/
boolean resetSupported();
/**
* Close input/ output streams if any
*/
void close();
}

View File

@ -115,9 +115,8 @@ public class TransformSplit extends BaseInputSplit {
* Close input/ output streams if any
*/
@Override
public void close() {
public void close() throws Exception {
sourceSplit.close();
}
public interface URITransform {

View File

@ -230,8 +230,10 @@ public class CSVSequenceRecordReaderTest extends BaseND4JTest {
}
@Override
public void close() throws Exception {
}
}

View File

@ -46,6 +46,11 @@ public class InputSplitTests extends BaseND4JTest {
@Test
public void testSample() throws URISyntaxException {
BaseInputSplit split = new BaseInputSplit() {
@Override
public void close() throws Exception {
}
{
String[] paths = {"label0/group1_img.tif", "label1/group1_img.jpg", "label2/group1_img.png",
"label3/group1_img.jpeg", "label4/group1_img.bmp", "label5/group1_img.JPEG",

View File

@ -18,19 +18,16 @@
* *****************************************************************************
*/
package org.datavec.api.split.parittion;
package org.datavec.api.split.partition;
import org.junit.jupiter.api.io.TempDir;
import org.nd4j.common.tests.BaseND4JTest;
import com.google.common.io.Files;
import org.datavec.api.conf.Configuration;
import org.datavec.api.split.FileSplit;
import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
import org.datavec.api.split.partition.PartitionMetaData;
import org.datavec.api.split.partition.Partitioner;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.OutputStream;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@ -38,9 +35,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class PartitionerTests extends BaseND4JTest {
@Test
public void testRecordsPerFilePartition() {
public void testRecordsPerFilePartition(@TempDir Path tmpDir) {
Partitioner partitioner = new NumberOfRecordsPartitioner();
File tmpDir = Files.createTempDir();
FileSplit fileSplit = new FileSplit(tmpDir);
assertTrue(fileSplit.needsBootstrapForWrite());
fileSplit.bootStrapForWrite();
@ -49,9 +45,8 @@ public class PartitionerTests extends BaseND4JTest {
}
@Test
public void testInputAddFile() throws Exception {
public void testInputAddFile(@TempDir Path tmpDir) throws Exception {
Partitioner partitioner = new NumberOfRecordsPartitioner();
File tmpDir = Files.createTempDir();
FileSplit fileSplit = new FileSplit(tmpDir);
assertTrue(fileSplit.needsBootstrapForWrite());
fileSplit.bootStrapForWrite();

View File

@ -154,11 +154,6 @@ public abstract class BaseAudioRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;

View File

@ -336,8 +336,7 @@ public class NativeImageLoader extends BaseImageLoader {
@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
throw new RuntimeException("Deprecated. Not implemented.");
/*Mat mat = streamToMat(inputStream);
Mat mat = streamToMat(inputStream);
Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
if (image == null || image.empty()) {
PIX pix = pixReadMem(mat.data(), mat.cols());
@ -348,14 +347,20 @@ public class NativeImageLoader extends BaseImageLoader {
pixDestroy(pix);
}
INDArray a = asMatrix(image);
if(!nchw)
a = a.permute(0,2,3,1); //NCHW to NHWC
if(!nchw) a = swapNCHWtoNHWC(a);
Image i = new Image(a, image.channels(), image.rows(), image.cols());
image.deallocate();
return i;
}
*/
/**
* Change channel order from NCHW to NHWC
* @param a input INDArray
* @return swapped INDArray
*/
private INDArray swapNCHWtoNHWC(INDArray a) {
return a.permute(0,2,3,1); //NCHW to NHWC
}
/**

View File

@ -405,8 +405,8 @@ public abstract class BaseImageRecordReader extends BaseRecordReader {
}
@Override
public void close() throws IOException {
this.inputSplit.close();
public void close() throws Exception {
inputSplit.close();
}
@Override

View File

@ -80,7 +80,7 @@ public class LoaderTests {
String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin";
String path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
byte[] fullDataExpected = new byte[3073];
FileInputStream inExpected = new FileInputStream(new File(path));
FileInputStream inExpected = new FileInputStream(path);
inExpected.read(fullDataExpected);
byte[] fullDataActual = new byte[3073];

View File

@ -45,6 +45,7 @@ import org.nd4j.common.io.ClassPathResource;
import java.io.File;
import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -54,18 +55,13 @@ import static org.junit.jupiter.api.Assertions.*;
public class TestObjectDetectionRecordReader {
@TempDir
public File testDir;
@Test
public void test() throws Exception {
public void test(@TempDir Path testDir) throws Exception {
for(boolean nchw : new boolean[]{true, false}) {
ImageObjectLabelProvider lp = new TestImageObjectDetectionLabelProvider();
File f = testDir;
new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(f);
String path = new File(f, "000012.jpg").getParent();
new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(testDir);
Path path = testDir.resolve("000012.jpg").getParent();
int h = 32;
int w = 32;
@ -74,7 +70,7 @@ public class TestObjectDetectionRecordReader {
int gH = 10;
//Enforce consistent iteration order for tests
URI[] u = new FileSplit(new File(path)).locations();
URI[] u = new FileSplit(path).locations();
Arrays.sort(u);
RecordReader rr = new ObjectDetectionRecordReader(h, w, c, gH, gW, nchw, lp);
@ -154,7 +150,7 @@ public class TestObjectDetectionRecordReader {
rr.reset();
Record record = rr.nextRecord();
RecordMetaDataImageURI metadata = (RecordMetaDataImageURI) record.getMetaData();
assertEquals(new File(path, "000012.jpg"), new File(metadata.getURI()));
assertEquals( path.resolve( "000012.jpg").toFile(), new File(metadata.getURI()));
assertEquals(3, metadata.getOrigC());
assertEquals((int) origH[0], metadata.getOrigH());
assertEquals((int) origW[0], metadata.getOrigW());

View File

@ -44,9 +44,7 @@ import java.lang.reflect.Array;
import java.util.*;
@Slf4j
@Builder
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
public class DynamicCustomOp extends DifferentialFunction implements CustomOp {
private String opName;
@ -56,7 +54,6 @@ public class DynamicCustomOp extends DifferentialFunction implements CustomOp {
@Builder.Default
protected List<INDArray> outputArguments = new ArrayList<>();
@Builder.Default
protected List<Double> tArguments = new ArrayList<>();

View File

@ -33,6 +33,7 @@ import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
@ -143,7 +144,6 @@ public class ClassPathResource extends AbstractFileResolvingResource {
public void copyDirectory(File destination) throws IOException {
Preconditions.checkState(destination.exists() && destination.isDirectory(), "Destination directory must exist and be a directory: %s", destination);
URL url = this.getUrl();
if (isJarURL(url)) {
@ -180,6 +180,7 @@ public class ClassPathResource extends AbstractFileResolvingResource {
try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(extractTo))){
InputStream is = getInputStream(name, clazz, classLoader);
IOUtils.copy(is, bos);
is.close();
}
}
}
@ -209,6 +210,9 @@ public class ClassPathResource extends AbstractFileResolvingResource {
}
}
public void copyDirectory(Path destination) throws IOException {
copyDirectory(destination.toFile());
}
public boolean exists() {
URL url;
if (this.clazz != null) {

View File

@ -27,4 +27,6 @@ dependencies {
implementation projects.cavisDnn.cavisDnnApi
implementation projects.cavisDatavec.cavisDatavecApi
implementation "commons-io:commons-io"
implementation "com.fasterxml.jackson.core:jackson-annotations"
implementation "com.fasterxml.jackson.core:jackson-core"
}