commit
ee67c022d0
|
@ -16,7 +16,7 @@ jobs:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
windows-x86_64:
|
windows-x86_64:
|
||||||
needs: pre-ci
|
needs: pre-ci
|
||||||
runs-on: windows-2019
|
runs-on: windows-2016
|
||||||
steps:
|
steps:
|
||||||
- name: Cancel Previous Runs
|
- name: Cancel Previous Runs
|
||||||
uses: styfle/cancel-workflow-action@0.8.0
|
uses: styfle/cancel-workflow-action@0.8.0
|
||||||
|
|
|
@ -12,7 +12,7 @@ DL4J was a junit 4 based code based for testing.
|
||||||
It's now based on junit 5's jupiter API, which has support for [Tags](https://junit.org/junit5/docs/5.0.1/api/org/junit/jupiter/api/Tag.html).
|
It's now based on junit 5's jupiter API, which has support for [Tags](https://junit.org/junit5/docs/5.0.1/api/org/junit/jupiter/api/Tag.html).
|
||||||
|
|
||||||
DL4j's code base has a number of different kinds of tests that fall in to several categories:
|
DL4j's code base has a number of different kinds of tests that fall in to several categories:
|
||||||
1. Long and flaky involving distributed systems (spark, parameter server)
|
1. Long and flaky involving distributed systems (spark, parameter-server)
|
||||||
2. Code that requires large downloads, but runs quickly
|
2. Code that requires large downloads, but runs quickly
|
||||||
3. Quick tests that test basic functionality
|
3. Quick tests that test basic functionality
|
||||||
4. Comprehensive integration tests that test several parts of a code base
|
4. Comprehensive integration tests that test several parts of a code base
|
||||||
|
@ -38,8 +38,10 @@ A few kinds of tags exist:
|
||||||
3. Distributed systems: spark, multi-threaded
|
3. Distributed systems: spark, multi-threaded
|
||||||
4. Functional cross-cutting concerns: multi module tests, similar functionality (excludes time based)
|
4. Functional cross-cutting concerns: multi module tests, similar functionality (excludes time based)
|
||||||
5. Platform specific tests that can vary on different hardware: cpu, gpu
|
5. Platform specific tests that can vary on different hardware: cpu, gpu
|
||||||
6. JVM crash: Tests with native code can crash the JVM for tests. It's useful to be able to turn those off when debugging.: jvm-crash
|
6. JVM crash: (jvm-crash) Tests with native code can crash the JVM for tests. It's useful to be able to turn those off when debugging.: jvm-crash
|
||||||
|
7. RNG: (rng) for RNG related tests
|
||||||
|
8. Samediff:(samediff) samediff related tests
|
||||||
|
9. Training related functionality
|
||||||
|
|
||||||
|
|
||||||
## Consequences
|
## Consequences
|
||||||
|
|
|
@ -124,6 +124,15 @@
|
||||||
</sources>
|
</sources>
|
||||||
</configuration>
|
</configuration>
|
||||||
</execution>
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>get-cpu-count</id>
|
||||||
|
<goals>
|
||||||
|
<goal>cpu-count</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<cpuCount>cpu.core.count</cpuCount>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
</executions>
|
</executions>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
|
||||||
|
|
|
@ -105,6 +105,12 @@
|
||||||
<artifactId>fastutil</artifactId>
|
<artifactId>fastutil</artifactId>
|
||||||
<version>${fastutil.version}</version>
|
<version>${fastutil.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.nd4j</groupId>
|
||||||
|
<artifactId>nd4j-common-tests</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<profiles>
|
<profiles>
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
@ -38,8 +39,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Csv Line Sequence Record Reader Test")
|
@DisplayName("Csv Line Sequence Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVLineSequenceRecordReaderTest extends BaseND4JTest {
|
class CSVLineSequenceRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
@ -54,8 +58,8 @@ class CSVLineSequenceRecordReaderTest extends BaseND4JTest {
|
||||||
FileUtils.writeStringToFile(source, str, StandardCharsets.UTF_8);
|
FileUtils.writeStringToFile(source, str, StandardCharsets.UTF_8);
|
||||||
SequenceRecordReader rr = new CSVLineSequenceRecordReader();
|
SequenceRecordReader rr = new CSVLineSequenceRecordReader();
|
||||||
rr.initialize(new FileSplit(source));
|
rr.initialize(new FileSplit(source));
|
||||||
List<List<Writable>> exp0 = Arrays.asList(Collections.<Writable>singletonList(new Text("a")), Collections.<Writable>singletonList(new Text("b")), Collections.<Writable>singletonList(new Text("c")));
|
List<List<Writable>> exp0 = Arrays.asList(Collections.singletonList(new Text("a")), Collections.singletonList(new Text("b")), Collections.<Writable>singletonList(new Text("c")));
|
||||||
List<List<Writable>> exp1 = Arrays.asList(Collections.<Writable>singletonList(new Text("1")), Collections.<Writable>singletonList(new Text("2")), Collections.<Writable>singletonList(new Text("3")), Collections.<Writable>singletonList(new Text("4")));
|
List<List<Writable>> exp1 = Arrays.asList(Collections.singletonList(new Text("1")), Collections.singletonList(new Text("2")), Collections.<Writable>singletonList(new Text("3")), Collections.<Writable>singletonList(new Text("4")));
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
int count = 0;
|
int count = 0;
|
||||||
while (rr.hasNext()) {
|
while (rr.hasNext()) {
|
||||||
|
|
|
@ -27,6 +27,7 @@ import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
@ -41,8 +42,11 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Csv Multi Sequence Record Reader Test")
|
@DisplayName("Csv Multi Sequence Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVMultiSequenceRecordReaderTest extends BaseND4JTest {
|
class CSVMultiSequenceRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.datavec.api.records.reader.impl.csv.CSVNLinesSequenceRecordReader;
|
||||||
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
|
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -34,12 +35,15 @@ import java.util.List;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Csvn Lines Sequence Record Reader Test")
|
@DisplayName("Csvn Lines Sequence Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVNLinesSequenceRecordReaderTest extends BaseND4JTest {
|
class CSVNLinesSequenceRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@DisplayName("Test CSVN Lines Sequence Record Reader")
|
@DisplayName("Test CSV Lines Sequence Record Reader")
|
||||||
void testCSVNLinesSequenceRecordReader() throws Exception {
|
void testCSVNLinesSequenceRecordReader() throws Exception {
|
||||||
int nLinesPerSequence = 10;
|
int nLinesPerSequence = 10;
|
||||||
SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence);
|
SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence);
|
||||||
|
|
|
@ -34,9 +34,12 @@ import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
|
@ -50,6 +53,8 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
|
||||||
@DisplayName("Csv Record Reader Test")
|
@DisplayName("Csv Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVRecordReaderTest extends BaseND4JTest {
|
class CSVRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -27,6 +27,7 @@ import org.datavec.api.split.InputSplit;
|
||||||
import org.datavec.api.split.NumberedFileInputSplit;
|
import org.datavec.api.split.NumberedFileInputSplit;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
@ -43,8 +44,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Csv Sequence Record Reader Test")
|
@DisplayName("Csv Sequence Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVSequenceRecordReaderTest extends BaseND4JTest {
|
class CSVSequenceRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -24,6 +24,7 @@ import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
|
||||||
import org.datavec.api.records.reader.impl.csv.CSVVariableSlidingWindowRecordReader;
|
import org.datavec.api.records.reader.impl.csv.CSVVariableSlidingWindowRecordReader;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -32,8 +33,11 @@ import java.util.List;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Csv Variable Sliding Window Record Reader Test")
|
@DisplayName("Csv Variable Sliding Window Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVVariableSlidingWindowRecordReaderTest extends BaseND4JTest {
|
class CSVVariableSlidingWindowRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader;
|
||||||
import org.datavec.api.records.reader.impl.filebatch.FileBatchSequenceRecordReader;
|
import org.datavec.api.records.reader.impl.filebatch.FileBatchSequenceRecordReader;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.junit.jupiter.params.ParameterizedTest;
|
import org.junit.jupiter.params.ParameterizedTest;
|
||||||
|
@ -42,9 +43,12 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.factory.Nd4jBackend;
|
import org.nd4j.linalg.factory.Nd4jBackend;
|
||||||
|
|
||||||
@DisplayName("File Batch Record Reader Test")
|
@DisplayName("File Batch Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class FileBatchRecordReaderTest extends BaseND4JTest {
|
public class FileBatchRecordReaderTest extends BaseND4JTest {
|
||||||
@TempDir Path testDir;
|
@TempDir Path testDir;
|
||||||
|
|
||||||
|
|
|
@ -25,6 +25,7 @@ import org.datavec.api.split.CollectionInputSplit;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.split.InputSplit;
|
import org.datavec.api.split.InputSplit;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -36,8 +37,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("File Record Reader Test")
|
@DisplayName("File Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class FileRecordReaderTest extends BaseND4JTest {
|
class FileRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -28,10 +28,12 @@ import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.shade.jackson.core.JsonFactory;
|
import org.nd4j.shade.jackson.core.JsonFactory;
|
||||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
@ -45,6 +47,8 @@ import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
|
||||||
@DisplayName("Jackson Line Record Reader Test")
|
@DisplayName("Jackson Line Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class JacksonLineRecordReaderTest extends BaseND4JTest {
|
class JacksonLineRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -31,10 +31,12 @@ import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.shade.jackson.core.JsonFactory;
|
import org.nd4j.shade.jackson.core.JsonFactory;
|
||||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||||
import org.nd4j.shade.jackson.dataformat.xml.XmlFactory;
|
import org.nd4j.shade.jackson.dataformat.xml.XmlFactory;
|
||||||
|
@ -51,6 +53,8 @@ import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
|
||||||
@DisplayName("Jackson Record Reader Test")
|
@DisplayName("Jackson Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class JacksonRecordReaderTest extends BaseND4JTest {
|
class JacksonRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.DoubleWritable;
|
import org.datavec.api.writable.DoubleWritable;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -35,9 +36,13 @@ import static org.datavec.api.records.reader.impl.misc.LibSvmRecordReader.*;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
@DisplayName("Lib Svm Record Reader Test")
|
@DisplayName("Lib Svm Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class LibSvmRecordReaderTest extends BaseND4JTest {
|
class LibSvmRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -30,6 +30,7 @@ import org.datavec.api.split.InputSplit;
|
||||||
import org.datavec.api.split.InputStreamInputSplit;
|
import org.datavec.api.split.InputStreamInputSplit;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
@ -47,8 +48,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Line Reader Test")
|
@DisplayName("Line Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class LineReaderTest extends BaseND4JTest {
|
class LineReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -33,6 +33,7 @@ import org.datavec.api.split.NumberedFileInputSplit;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
@ -46,8 +47,11 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Regex Record Reader Test")
|
@DisplayName("Regex Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class RegexRecordReaderTest extends BaseND4JTest {
|
class RegexRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.DoubleWritable;
|
import org.datavec.api.writable.DoubleWritable;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -35,9 +36,13 @@ import static org.datavec.api.records.reader.impl.misc.SVMLightRecordReader.*;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
@DisplayName("Svm Light Record Reader Test")
|
@DisplayName("Svm Light Record Reader Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class SVMLightRecordReaderTest extends BaseND4JTest {
|
class SVMLightRecordReaderTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -26,15 +26,18 @@ import org.datavec.api.records.reader.SequenceRecordReader;
|
||||||
import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader;
|
import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestCollectionRecordReaders extends BaseND4JTest {
|
public class TestCollectionRecordReaders extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -23,12 +23,15 @@ package org.datavec.api.records.reader.impl;
|
||||||
import org.datavec.api.records.reader.RecordReader;
|
import org.datavec.api.records.reader.RecordReader;
|
||||||
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
|
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestConcatenatingRecordReader extends BaseND4JTest {
|
public class TestConcatenatingRecordReader extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -37,9 +37,11 @@ import org.datavec.api.transform.TransformProcess;
|
||||||
import org.datavec.api.transform.schema.Schema;
|
import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.shade.jackson.core.JsonFactory;
|
import org.nd4j.shade.jackson.core.JsonFactory;
|
||||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
@ -48,7 +50,8 @@ import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestSerialization extends BaseND4JTest {
|
public class TestSerialization extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -30,9 +30,11 @@ import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.LongWritable;
|
import org.datavec.api.writable.LongWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -41,6 +43,8 @@ import java.util.List;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TransformProcessRecordReaderTests extends BaseND4JTest {
|
public class TransformProcessRecordReaderTests extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -74,11 +78,11 @@ public class TransformProcessRecordReaderTests extends BaseND4JTest {
|
||||||
public void simpleTransformTestSequence() {
|
public void simpleTransformTestSequence() {
|
||||||
List<List<Writable>> sequence = new ArrayList<>();
|
List<List<Writable>> sequence = new ArrayList<>();
|
||||||
//First window:
|
//First window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0),
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0),
|
||||||
new IntWritable(0)));
|
new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1),
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1),
|
||||||
new IntWritable(0)));
|
new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2),
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2),
|
||||||
new IntWritable(0)));
|
new IntWritable(0)));
|
||||||
|
|
||||||
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
@ -34,8 +35,11 @@ import java.util.List;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Csv Record Writer Test")
|
@DisplayName("Csv Record Writer Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class CSVRecordWriterTest extends BaseND4JTest {
|
class CSVRecordWriterTest extends BaseND4JTest {
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
|
|
|
@ -29,8 +29,10 @@ import org.datavec.api.writable.DoubleWritable;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.NDArrayWritable;
|
import org.datavec.api.writable.NDArrayWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -46,6 +48,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
@DisplayName("Lib Svm Record Writer Test")
|
@DisplayName("Lib Svm Record Writer Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class LibSvmRecordWriterTest extends BaseND4JTest {
|
class LibSvmRecordWriterTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -26,8 +26,10 @@ import org.datavec.api.records.writer.impl.misc.SVMLightRecordWriter;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
|
import org.datavec.api.split.partition.NumberOfRecordsPartitioner;
|
||||||
import org.datavec.api.writable.*;
|
import org.datavec.api.writable.*;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -43,6 +45,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
@DisplayName("Svm Light Record Writer Test")
|
@DisplayName("Svm Light Record Writer Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class SVMLightRecordWriterTest extends BaseND4JTest {
|
class SVMLightRecordWriterTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -20,7 +20,9 @@
|
||||||
|
|
||||||
package org.datavec.api.split;
|
package org.datavec.api.split;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.shade.guava.io.Files;
|
import org.nd4j.shade.guava.io.Files;
|
||||||
import org.datavec.api.io.filters.BalancedPathFilter;
|
import org.datavec.api.io.filters.BalancedPathFilter;
|
||||||
import org.datavec.api.io.filters.RandomPathFilter;
|
import org.datavec.api.io.filters.RandomPathFilter;
|
||||||
|
@ -42,6 +44,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
*
|
*
|
||||||
* @author saudet
|
* @author saudet
|
||||||
*/
|
*/
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class InputSplitTests extends BaseND4JTest {
|
public class InputSplitTests extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -20,13 +20,16 @@
|
||||||
|
|
||||||
package org.datavec.api.split;
|
package org.datavec.api.split;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class NumberedFileInputSplitTests extends BaseND4JTest {
|
public class NumberedFileInputSplitTests extends BaseND4JTest {
|
||||||
@Test
|
@Test
|
||||||
public void testNumberedFileInputSplitBasic() {
|
public void testNumberedFileInputSplitBasic() {
|
||||||
|
|
|
@ -26,11 +26,13 @@ import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.function.Function;
|
import org.nd4j.common.function.Function;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
|
@ -46,7 +48,8 @@ import java.util.Random;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestStreamInputSplit extends BaseND4JTest {
|
public class TestStreamInputSplit extends BaseND4JTest {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
*/
|
*/
|
||||||
package org.datavec.api.split;
|
package org.datavec.api.split;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
@ -28,11 +29,14 @@ import static java.util.Arrays.asList;
|
||||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author Ede Meijer
|
* @author Ede Meijer
|
||||||
*/
|
*/
|
||||||
@DisplayName("Transform Split Test")
|
@DisplayName("Transform Split Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class TransformSplitTest extends BaseND4JTest {
|
class TransformSplitTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -20,7 +20,9 @@
|
||||||
|
|
||||||
package org.datavec.api.split.parittion;
|
package org.datavec.api.split.parittion;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.shade.guava.io.Files;
|
import org.nd4j.shade.guava.io.Files;
|
||||||
import org.datavec.api.conf.Configuration;
|
import org.datavec.api.conf.Configuration;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
|
@ -33,7 +35,8 @@ import java.io.File;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class PartitionerTests extends BaseND4JTest {
|
public class PartitionerTests extends BaseND4JTest {
|
||||||
@Test
|
@Test
|
||||||
public void testRecordsPerFilePartition() {
|
public void testRecordsPerFilePartition() {
|
||||||
|
|
|
@ -29,13 +29,16 @@ import org.datavec.api.writable.DoubleWritable;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestTransformProcess extends BaseND4JTest {
|
public class TestTransformProcess extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -27,14 +27,17 @@ import org.datavec.api.transform.condition.string.StringRegexColumnCondition;
|
||||||
import org.datavec.api.transform.schema.Schema;
|
import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.transform.transform.TestTransforms;
|
import org.datavec.api.transform.transform.TestTransforms;
|
||||||
import org.datavec.api.writable.*;
|
import org.datavec.api.writable.*;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestConditions extends BaseND4JTest {
|
public class TestConditions extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -27,8 +27,10 @@ import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.writable.DoubleWritable;
|
import org.datavec.api.writable.DoubleWritable;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -38,7 +40,8 @@ import java.util.List;
|
||||||
import static java.util.Arrays.asList;
|
import static java.util.Arrays.asList;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestFilters extends BaseND4JTest {
|
public class TestFilters extends BaseND4JTest {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -26,9 +26,11 @@ import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.NullWritable;
|
import org.datavec.api.writable.NullWritable;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -37,7 +39,8 @@ import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestJoin extends BaseND4JTest {
|
public class TestJoin extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -33,14 +33,17 @@ import org.datavec.api.transform.ops.IAggregableReduceOp;
|
||||||
import org.datavec.api.transform.schema.Schema;
|
import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.writable.*;
|
import org.datavec.api.writable.*;
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.fail;
|
import static org.junit.jupiter.api.Assertions.fail;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestMultiOpReduce extends BaseND4JTest {
|
public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -91,10 +94,10 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
public void testReducerInteger() {
|
public void testReducerInteger() {
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(0)));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(0)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1)));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2)));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2)));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2)));
|
||||||
|
|
||||||
Map<ReduceOp, Double> exp = new LinkedHashMap<>();
|
Map<ReduceOp, Double> exp = new LinkedHashMap<>();
|
||||||
exp.put(ReduceOp.Min, 0.0);
|
exp.put(ReduceOp.Min, 0.0);
|
||||||
|
@ -133,9 +136,7 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@Disabled
|
|
||||||
public void testReduceString() {
|
public void testReduceString() {
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.asList(new Text("someKey"), new Text("1")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("1")));
|
||||||
inputs.add(Arrays.asList(new Text("someKey"), new Text("2")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("2")));
|
||||||
|
@ -165,7 +166,7 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
assertEquals(out.get(0), new Text("someKey"));
|
assertEquals(out.get(0), new Text("someKey"));
|
||||||
|
|
||||||
String msg = op.toString();
|
String msg = op.toString();
|
||||||
assertEquals(msg, exp.get(op), out.get(1).toString());
|
assertEquals(exp.get(op), out.get(1).toString(),msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -173,12 +174,12 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
public void testReduceIntegerIgnoreInvalidValues() {
|
public void testReduceIntegerIgnoreInvalidValues() {
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("0")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("0")));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("1")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("1")));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2)));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("ignore me")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("ignore me")));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("also ignore me")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("also ignore me")));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("2")));
|
inputs.add(Arrays.asList(new Text("someKey"), new Text("2")));
|
||||||
|
|
||||||
|
|
||||||
Map<ReduceOp, Double> exp = new LinkedHashMap<>();
|
Map<ReduceOp, Double> exp = new LinkedHashMap<>();
|
||||||
|
@ -240,16 +241,16 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
public void testCustomReductions() {
|
public void testCustomReductions() {
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1), new Text("zero"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("zero"),
|
||||||
new DoubleWritable(0)));
|
new DoubleWritable(0)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2), new Text("one"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("one"),
|
||||||
new DoubleWritable(1)));
|
new DoubleWritable(1)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(3), new Text("two"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("two"),
|
||||||
new DoubleWritable(2)));
|
new DoubleWritable(2)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(4), new Text("three"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("three"),
|
||||||
new DoubleWritable(3)));
|
new DoubleWritable(3)));
|
||||||
|
|
||||||
List<Writable> expected = Arrays.asList((Writable) new Text("someKey"), new IntWritable(10), new Text("one"),
|
List<Writable> expected = Arrays.asList(new Text("someKey"), new IntWritable(10), new Text("one"),
|
||||||
new DoubleWritable(1));
|
new DoubleWritable(1));
|
||||||
|
|
||||||
|
|
||||||
|
@ -290,16 +291,16 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
public void testCustomReductionsWithCondition() {
|
public void testCustomReductionsWithCondition() {
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1), new Text("zero"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("zero"),
|
||||||
new DoubleWritable(0)));
|
new DoubleWritable(0)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2), new Text("one"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("one"),
|
||||||
new DoubleWritable(1)));
|
new DoubleWritable(1)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(3), new Text("two"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("two"),
|
||||||
new DoubleWritable(2)));
|
new DoubleWritable(2)));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(4), new Text("three"),
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("three"),
|
||||||
new DoubleWritable(3)));
|
new DoubleWritable(3)));
|
||||||
|
|
||||||
List<Writable> expected = Arrays.asList((Writable) new Text("someKey"), new IntWritable(10), new IntWritable(3),
|
List<Writable> expected = Arrays.asList(new Text("someKey"), new IntWritable(10), new IntWritable(3),
|
||||||
new DoubleWritable(1));
|
new DoubleWritable(1));
|
||||||
|
|
||||||
|
|
||||||
|
@ -343,7 +344,7 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
public IAggregableReduceOp<Writable, List<Writable>> reduceOp() {
|
public IAggregableReduceOp<Writable, List<Writable>> reduceOp() {
|
||||||
//For testing: let's take the second value
|
//For testing: let's take the second value
|
||||||
return new AggregableMultiOp<>(Collections
|
return new AggregableMultiOp<>(Collections
|
||||||
.<IAggregableReduceOp<Writable, Writable>>singletonList(new AggregableSecond<Writable>()));
|
.<IAggregableReduceOp<Writable, Writable>>singletonList(new AggregableSecond<>()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -485,12 +486,12 @@ public class TestMultiOpReduce extends BaseND4JTest {
|
||||||
.addColumnString("filterCol").addColumnString("textCol").build();
|
.addColumnString("filterCol").addColumnString("textCol").build();
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(1), new Text("a"), new Text("zero")));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("a"), new Text("zero")));
|
||||||
inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(2), new Text("b"), new Text("one")));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("b"), new Text("one")));
|
||||||
inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(3), new Text("a"), new Text("two")));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("a"), new Text("two")));
|
||||||
inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(4), new Text("b"), new Text("three")));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("b"), new Text("three")));
|
||||||
inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(5), new Text("a"), new Text("three")));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(5), new Text("a"), new Text("three")));
|
||||||
inputs.add(Arrays.<Writable>asList(new Text("someKey"), new IntWritable(6), new Text("b"), new Text("three")));
|
inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(6), new Text("b"), new Text("three")));
|
||||||
|
|
||||||
Condition condition = new StringColumnCondition("filterCol", ConditionOp.Equal, "a");
|
Condition condition = new StringColumnCondition("filterCol", ConditionOp.Equal, "a");
|
||||||
|
|
||||||
|
|
|
@ -24,14 +24,17 @@ import org.datavec.api.transform.ops.IAggregableReduceOp;
|
||||||
import org.datavec.api.transform.reduce.impl.GeographicMidpointReduction;
|
import org.datavec.api.transform.reduce.impl.GeographicMidpointReduction;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestReductions extends BaseND4JTest {
|
public class TestReductions extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -22,11 +22,15 @@ package org.datavec.api.transform.schema;
|
||||||
|
|
||||||
import org.datavec.api.transform.metadata.ColumnMetaData;
|
import org.datavec.api.transform.metadata.ColumnMetaData;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JACKSON_SERDE)
|
||||||
public class TestJsonYaml extends BaseND4JTest {
|
public class TestJsonYaml extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -21,11 +21,14 @@
|
||||||
package org.datavec.api.transform.schema;
|
package org.datavec.api.transform.schema;
|
||||||
|
|
||||||
import org.datavec.api.transform.ColumnType;
|
import org.datavec.api.transform.ColumnType;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestSchemaMethods extends BaseND4JTest {
|
public class TestSchemaMethods extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -33,8 +33,10 @@ import org.datavec.api.writable.LongWritable;
|
||||||
import org.datavec.api.writable.NullWritable;
|
import org.datavec.api.writable.NullWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -42,7 +44,8 @@ import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestReduceSequenceByWindowFunction extends BaseND4JTest {
|
public class TestReduceSequenceByWindowFunction extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -27,8 +27,10 @@ import org.datavec.api.writable.LongWritable;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -36,7 +38,8 @@ import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestSequenceSplit extends BaseND4JTest {
|
public class TestSequenceSplit extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -46,13 +49,13 @@ public class TestSequenceSplit extends BaseND4JTest {
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
List<List<Writable>> inputSequence = new ArrayList<>();
|
List<List<Writable>> inputSequence = new ArrayList<>();
|
||||||
inputSequence.add(Arrays.asList((Writable) new LongWritable(0), new Text("t0")));
|
inputSequence.add(Arrays.asList(new LongWritable(0), new Text("t0")));
|
||||||
inputSequence.add(Arrays.asList((Writable) new LongWritable(1000), new Text("t1")));
|
inputSequence.add(Arrays.asList(new LongWritable(1000), new Text("t1")));
|
||||||
//Second split: 74 seconds later
|
//Second split: 74 seconds later
|
||||||
inputSequence.add(Arrays.asList((Writable) new LongWritable(75000), new Text("t2")));
|
inputSequence.add(Arrays.asList(new LongWritable(75000), new Text("t2")));
|
||||||
inputSequence.add(Arrays.asList((Writable) new LongWritable(100000), new Text("t3")));
|
inputSequence.add(Arrays.asList(new LongWritable(100000), new Text("t3")));
|
||||||
//Third split: 1 minute and 1 milliseconds later
|
//Third split: 1 minute and 1 milliseconds later
|
||||||
inputSequence.add(Arrays.asList((Writable) new LongWritable(160001), new Text("t4")));
|
inputSequence.add(Arrays.asList(new LongWritable(160001), new Text("t4")));
|
||||||
|
|
||||||
SequenceSplit seqSplit = new SequenceSplitTimeSeparation("time", 1, TimeUnit.MINUTES);
|
SequenceSplit seqSplit = new SequenceSplitTimeSeparation("time", 1, TimeUnit.MINUTES);
|
||||||
seqSplit.setInputSchema(schema);
|
seqSplit.setInputSchema(schema);
|
||||||
|
@ -61,13 +64,13 @@ public class TestSequenceSplit extends BaseND4JTest {
|
||||||
assertEquals(3, splits.size());
|
assertEquals(3, splits.size());
|
||||||
|
|
||||||
List<List<Writable>> exp0 = new ArrayList<>();
|
List<List<Writable>> exp0 = new ArrayList<>();
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(0), new Text("t0")));
|
exp0.add(Arrays.asList(new LongWritable(0), new Text("t0")));
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(1000), new Text("t1")));
|
exp0.add(Arrays.asList(new LongWritable(1000), new Text("t1")));
|
||||||
List<List<Writable>> exp1 = new ArrayList<>();
|
List<List<Writable>> exp1 = new ArrayList<>();
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(75000), new Text("t2")));
|
exp1.add(Arrays.asList(new LongWritable(75000), new Text("t2")));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(100000), new Text("t3")));
|
exp1.add(Arrays.asList(new LongWritable(100000), new Text("t3")));
|
||||||
List<List<Writable>> exp2 = new ArrayList<>();
|
List<List<Writable>> exp2 = new ArrayList<>();
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(160001), new Text("t4")));
|
exp2.add(Arrays.asList(new LongWritable(160001), new Text("t4")));
|
||||||
|
|
||||||
assertEquals(exp0, splits.get(0));
|
assertEquals(exp0, splits.get(0));
|
||||||
assertEquals(exp1, splits.get(1));
|
assertEquals(exp1, splits.get(1));
|
||||||
|
|
|
@ -29,8 +29,10 @@ import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.LongWritable;
|
import org.datavec.api.writable.LongWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -38,7 +40,8 @@ import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestWindowFunctions extends BaseND4JTest {
|
public class TestWindowFunctions extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -49,15 +52,15 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
//Create some data.
|
//Create some data.
|
||||||
List<List<Writable>> sequence = new ArrayList<>();
|
List<List<Writable>> sequence = new ArrayList<>();
|
||||||
//First window:
|
//First window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2)));
|
||||||
//Second window:
|
//Second window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 1000L), new IntWritable(3)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 1000L), new IntWritable(3)));
|
||||||
//Third window: empty
|
//Third window: empty
|
||||||
//Fourth window:
|
//Fourth window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3000L), new IntWritable(4)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3000L), new IntWritable(4)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3100L), new IntWritable(5)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3100L), new IntWritable(5)));
|
||||||
|
|
||||||
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
||||||
.addColumnInteger("intcolumn").build();
|
.addColumnInteger("intcolumn").build();
|
||||||
|
@ -100,15 +103,15 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
//Create some data.
|
//Create some data.
|
||||||
List<List<Writable>> sequence = new ArrayList<>();
|
List<List<Writable>> sequence = new ArrayList<>();
|
||||||
//First window:
|
//First window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2)));
|
||||||
//Second window:
|
//Second window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 1000L), new IntWritable(3)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 1000L), new IntWritable(3)));
|
||||||
//Third window: empty
|
//Third window: empty
|
||||||
//Fourth window:
|
//Fourth window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3000L), new IntWritable(4)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3000L), new IntWritable(4)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3100L), new IntWritable(5)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3100L), new IntWritable(5)));
|
||||||
|
|
||||||
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
||||||
.addColumnInteger("intcolumn").build();
|
.addColumnInteger("intcolumn").build();
|
||||||
|
@ -150,15 +153,15 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
//Create some data.
|
//Create some data.
|
||||||
List<List<Writable>> sequence = new ArrayList<>();
|
List<List<Writable>> sequence = new ArrayList<>();
|
||||||
//First window:
|
//First window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2)));
|
||||||
//Second window:
|
//Second window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 1000L), new IntWritable(3)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 1000L), new IntWritable(3)));
|
||||||
//Third window: empty
|
//Third window: empty
|
||||||
//Fourth window:
|
//Fourth window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3000L), new IntWritable(4)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3000L), new IntWritable(4)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3100L), new IntWritable(5)));
|
sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3100L), new IntWritable(5)));
|
||||||
|
|
||||||
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
||||||
.addColumnInteger("intcolumn").build();
|
.addColumnInteger("intcolumn").build();
|
||||||
|
@ -188,13 +191,13 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
//Create some data.
|
//Create some data.
|
||||||
List<List<Writable>> sequence = new ArrayList<>();
|
List<List<Writable>> sequence = new ArrayList<>();
|
||||||
//First window:
|
//First window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0)));
|
sequence.add(Arrays.asList(new LongWritable(0), new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1)));
|
sequence.add(Arrays.asList(new LongWritable(100), new IntWritable(1)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2)));
|
sequence.add(Arrays.asList(new LongWritable(200), new IntWritable(2)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3)));
|
sequence.add(Arrays.asList(new LongWritable(1000), new IntWritable(3)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4)));
|
sequence.add(Arrays.asList(new LongWritable(1500), new IntWritable(4)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5)));
|
sequence.add(Arrays.asList(new LongWritable(2000), new IntWritable(5)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7)));
|
sequence.add(Arrays.asList(new LongWritable(5000), new IntWritable(7)));
|
||||||
|
|
||||||
|
|
||||||
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
||||||
|
@ -207,32 +210,32 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
|
|
||||||
//First window: -1000 to 1000
|
//First window: -1000 to 1000
|
||||||
List<List<Writable>> exp0 = new ArrayList<>();
|
List<List<Writable>> exp0 = new ArrayList<>();
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0)));
|
exp0.add(Arrays.asList(new LongWritable(0), new IntWritable(0)));
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1)));
|
exp0.add(Arrays.asList(new LongWritable(100), new IntWritable(1)));
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2)));
|
exp0.add(Arrays.asList(new LongWritable(200), new IntWritable(2)));
|
||||||
//Second window: 0 to 2000
|
//Second window: 0 to 2000
|
||||||
List<List<Writable>> exp1 = new ArrayList<>();
|
List<List<Writable>> exp1 = new ArrayList<>();
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0)));
|
exp1.add(Arrays.asList(new LongWritable(0), new IntWritable(0)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1)));
|
exp1.add(Arrays.asList(new LongWritable(100), new IntWritable(1)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2)));
|
exp1.add(Arrays.asList(new LongWritable(200), new IntWritable(2)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3)));
|
exp1.add(Arrays.asList(new LongWritable(1000), new IntWritable(3)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4)));
|
exp1.add(Arrays.asList(new LongWritable(1500), new IntWritable(4)));
|
||||||
//Third window: 1000 to 3000
|
//Third window: 1000 to 3000
|
||||||
List<List<Writable>> exp2 = new ArrayList<>();
|
List<List<Writable>> exp2 = new ArrayList<>();
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3)));
|
exp2.add(Arrays.asList(new LongWritable(1000), new IntWritable(3)));
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4)));
|
exp2.add(Arrays.asList(new LongWritable(1500), new IntWritable(4)));
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5)));
|
exp2.add(Arrays.asList(new LongWritable(2000), new IntWritable(5)));
|
||||||
//Fourth window: 2000 to 4000
|
//Fourth window: 2000 to 4000
|
||||||
List<List<Writable>> exp3 = new ArrayList<>();
|
List<List<Writable>> exp3 = new ArrayList<>();
|
||||||
exp3.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5)));
|
exp3.add(Arrays.asList(new LongWritable(2000), new IntWritable(5)));
|
||||||
//Fifth window: 3000 to 5000
|
//Fifth window: 3000 to 5000
|
||||||
List<List<Writable>> exp4 = new ArrayList<>();
|
List<List<Writable>> exp4 = new ArrayList<>();
|
||||||
//Sixth window: 4000 to 6000
|
//Sixth window: 4000 to 6000
|
||||||
List<List<Writable>> exp5 = new ArrayList<>();
|
List<List<Writable>> exp5 = new ArrayList<>();
|
||||||
exp5.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7)));
|
exp5.add(Arrays.asList(new LongWritable(5000), new IntWritable(7)));
|
||||||
//Seventh window: 5000 to 7000
|
//Seventh window: 5000 to 7000
|
||||||
List<List<Writable>> exp6 = new ArrayList<>();
|
List<List<Writable>> exp6 = new ArrayList<>();
|
||||||
exp6.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7)));
|
exp6.add(Arrays.asList(new LongWritable(5000), new IntWritable(7)));
|
||||||
|
|
||||||
List<List<List<Writable>>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp4, exp5, exp6);
|
List<List<List<Writable>>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp4, exp5, exp6);
|
||||||
|
|
||||||
|
@ -250,13 +253,13 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
//Create some data.
|
//Create some data.
|
||||||
List<List<Writable>> sequence = new ArrayList<>();
|
List<List<Writable>> sequence = new ArrayList<>();
|
||||||
//First window:
|
//First window:
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0)));
|
sequence.add(Arrays.asList(new LongWritable(0), new IntWritable(0)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1)));
|
sequence.add(Arrays.asList(new LongWritable(100), new IntWritable(1)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2)));
|
sequence.add(Arrays.asList(new LongWritable(200), new IntWritable(2)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3)));
|
sequence.add(Arrays.asList(new LongWritable(1000), new IntWritable(3)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4)));
|
sequence.add(Arrays.asList(new LongWritable(1500), new IntWritable(4)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5)));
|
sequence.add(Arrays.asList(new LongWritable(2000), new IntWritable(5)));
|
||||||
sequence.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7)));
|
sequence.add(Arrays.asList(new LongWritable(5000), new IntWritable(7)));
|
||||||
|
|
||||||
|
|
||||||
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC)
|
||||||
|
@ -272,31 +275,31 @@ public class TestWindowFunctions extends BaseND4JTest {
|
||||||
|
|
||||||
//First window: -1000 to 1000
|
//First window: -1000 to 1000
|
||||||
List<List<Writable>> exp0 = new ArrayList<>();
|
List<List<Writable>> exp0 = new ArrayList<>();
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0)));
|
exp0.add(Arrays.asList(new LongWritable(0), new IntWritable(0)));
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1)));
|
exp0.add(Arrays.asList(new LongWritable(100), new IntWritable(1)));
|
||||||
exp0.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2)));
|
exp0.add(Arrays.asList(new LongWritable(200), new IntWritable(2)));
|
||||||
//Second window: 0 to 2000
|
//Second window: 0 to 2000
|
||||||
List<List<Writable>> exp1 = new ArrayList<>();
|
List<List<Writable>> exp1 = new ArrayList<>();
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0)));
|
exp1.add(Arrays.asList(new LongWritable(0), new IntWritable(0)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1)));
|
exp1.add(Arrays.asList(new LongWritable(100), new IntWritable(1)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2)));
|
exp1.add(Arrays.asList(new LongWritable(200), new IntWritable(2)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3)));
|
exp1.add(Arrays.asList(new LongWritable(1000), new IntWritable(3)));
|
||||||
exp1.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4)));
|
exp1.add(Arrays.asList(new LongWritable(1500), new IntWritable(4)));
|
||||||
//Third window: 1000 to 3000
|
//Third window: 1000 to 3000
|
||||||
List<List<Writable>> exp2 = new ArrayList<>();
|
List<List<Writable>> exp2 = new ArrayList<>();
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3)));
|
exp2.add(Arrays.asList(new LongWritable(1000), new IntWritable(3)));
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4)));
|
exp2.add(Arrays.asList(new LongWritable(1500), new IntWritable(4)));
|
||||||
exp2.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5)));
|
exp2.add(Arrays.asList(new LongWritable(2000), new IntWritable(5)));
|
||||||
//Fourth window: 2000 to 4000
|
//Fourth window: 2000 to 4000
|
||||||
List<List<Writable>> exp3 = new ArrayList<>();
|
List<List<Writable>> exp3 = new ArrayList<>();
|
||||||
exp3.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5)));
|
exp3.add(Arrays.asList(new LongWritable(2000), new IntWritable(5)));
|
||||||
//Fifth window: 3000 to 5000 -> Empty: excluded
|
//Fifth window: 3000 to 5000 -> Empty: excluded
|
||||||
//Sixth window: 4000 to 6000
|
//Sixth window: 4000 to 6000
|
||||||
List<List<Writable>> exp5 = new ArrayList<>();
|
List<List<Writable>> exp5 = new ArrayList<>();
|
||||||
exp5.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7)));
|
exp5.add(Arrays.asList(new LongWritable(5000), new IntWritable(7)));
|
||||||
//Seventh window: 5000 to 7000
|
//Seventh window: 5000 to 7000
|
||||||
List<List<Writable>> exp6 = new ArrayList<>();
|
List<List<Writable>> exp6 = new ArrayList<>();
|
||||||
exp6.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7)));
|
exp6.add(Arrays.asList(new LongWritable(5000), new IntWritable(7)));
|
||||||
|
|
||||||
List<List<List<Writable>>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp5, exp6);
|
List<List<List<Writable>>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp5, exp6);
|
||||||
|
|
||||||
|
|
|
@ -26,11 +26,17 @@ import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.transform.serde.testClasses.CustomCondition;
|
import org.datavec.api.transform.serde.testClasses.CustomCondition;
|
||||||
import org.datavec.api.transform.serde.testClasses.CustomFilter;
|
import org.datavec.api.transform.serde.testClasses.CustomFilter;
|
||||||
import org.datavec.api.transform.serde.testClasses.CustomTransform;
|
import org.datavec.api.transform.serde.testClasses.CustomTransform;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JACKSON_SERDE)
|
||||||
|
@Tag(TagNames.CUSTOM_FUNCTIONALITY)
|
||||||
public class TestCustomTransformJsonYaml extends BaseND4JTest {
|
public class TestCustomTransformJsonYaml extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -64,14 +64,19 @@ import org.datavec.api.transform.transform.time.TimeMathOpTransform;
|
||||||
import org.datavec.api.writable.comparator.DoubleWritableComparator;
|
import org.datavec.api.writable.comparator.DoubleWritableComparator;
|
||||||
import org.joda.time.DateTimeFieldType;
|
import org.joda.time.DateTimeFieldType;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JACKSON_SERDE)
|
||||||
public class TestYamlJsonSerde extends BaseND4JTest {
|
public class TestYamlJsonSerde extends BaseND4JTest {
|
||||||
|
|
||||||
public static YamlSerializer y = new YamlSerializer();
|
public static YamlSerializer y = new YamlSerializer();
|
||||||
|
|
|
@ -24,22 +24,26 @@ import org.datavec.api.transform.StringReduceOp;
|
||||||
import org.datavec.api.transform.schema.Schema;
|
import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.writable.Text;
|
import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestReduce extends BaseND4JTest {
|
public class TestReduce extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testReducerDouble() {
|
public void testReducerDouble() {
|
||||||
|
|
||||||
List<List<Writable>> inputs = new ArrayList<>();
|
List<List<Writable>> inputs = new ArrayList<>();
|
||||||
inputs.add(Arrays.asList((Writable) new Text("1"), new Text("2")));
|
inputs.add(Arrays.asList(new Text("1"), new Text("2")));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("1"), new Text("2")));
|
inputs.add(Arrays.asList(new Text("1"), new Text("2")));
|
||||||
inputs.add(Arrays.asList((Writable) new Text("1"), new Text("2")));
|
inputs.add(Arrays.asList(new Text("1"), new Text("2")));
|
||||||
|
|
||||||
Map<StringReduceOp, String> exp = new LinkedHashMap<>();
|
Map<StringReduceOp, String> exp = new LinkedHashMap<>();
|
||||||
exp.put(StringReduceOp.MERGE, "12");
|
exp.put(StringReduceOp.MERGE, "12");
|
||||||
|
|
|
@ -37,10 +37,12 @@ import org.datavec.api.writable.Writable;
|
||||||
import org.joda.time.DateTimeZone;
|
import org.joda.time.DateTimeZone;
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
@ -49,7 +51,9 @@ import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.UI)
|
||||||
public class TestUI extends BaseND4JTest {
|
public class TestUI extends BaseND4JTest {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
package org.datavec.api.util;
|
package org.datavec.api.util;
|
||||||
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
|
@ -33,8 +34,11 @@ import static org.hamcrest.core.AnyOf.anyOf;
|
||||||
import static org.hamcrest.core.IsEqual.equalTo;
|
import static org.hamcrest.core.IsEqual.equalTo;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Class Path Resource Test")
|
@DisplayName("Class Path Resource Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class ClassPathResourceTest extends BaseND4JTest {
|
class ClassPathResourceTest extends BaseND4JTest {
|
||||||
|
|
||||||
// File sizes are reported slightly different on Linux vs. Windows
|
// File sizes are reported slightly different on Linux vs. Windows
|
||||||
|
|
|
@ -22,8 +22,10 @@ package org.datavec.api.util;
|
||||||
import org.datavec.api.timeseries.util.TimeSeriesWritableUtils;
|
import org.datavec.api.timeseries.util.TimeSeriesWritableUtils;
|
||||||
import org.datavec.api.writable.DoubleWritable;
|
import org.datavec.api.writable.DoubleWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -32,6 +34,8 @@ import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
|
||||||
@DisplayName("Time Series Utils Test")
|
@DisplayName("Time Series Utils Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class TimeSeriesUtilsTest extends BaseND4JTest {
|
class TimeSeriesUtilsTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -19,7 +19,9 @@
|
||||||
*/
|
*/
|
||||||
package org.datavec.api.writable;
|
package org.datavec.api.writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.shade.guava.collect.Lists;
|
import org.nd4j.shade.guava.collect.Lists;
|
||||||
import org.datavec.api.transform.schema.Schema;
|
import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.util.ndarray.RecordConverter;
|
import org.datavec.api.util.ndarray.RecordConverter;
|
||||||
|
@ -36,6 +38,8 @@ import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
|
||||||
@DisplayName("Record Converter Test")
|
@DisplayName("Record Converter Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class RecordConverterTest extends BaseND4JTest {
|
class RecordConverterTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -21,15 +21,18 @@
|
||||||
package org.datavec.api.writable;
|
package org.datavec.api.writable;
|
||||||
|
|
||||||
import org.datavec.api.transform.metadata.NDArrayMetaData;
|
import org.datavec.api.transform.metadata.NDArrayMetaData;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestNDArrayWritableAndSerialization extends BaseND4JTest {
|
public class TestNDArrayWritableAndSerialization extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -20,8 +20,10 @@
|
||||||
package org.datavec.api.writable;
|
package org.datavec.api.writable;
|
||||||
|
|
||||||
import org.datavec.api.writable.batch.NDArrayRecordBatch;
|
import org.datavec.api.writable.batch.NDArrayRecordBatch;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.buffer.DataBuffer;
|
import org.nd4j.linalg.api.buffer.DataBuffer;
|
||||||
import org.nd4j.linalg.api.buffer.DataType;
|
import org.nd4j.linalg.api.buffer.DataType;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
|
@ -37,6 +39,8 @@ import org.junit.jupiter.api.DisplayName;
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
@DisplayName("Writable Test")
|
@DisplayName("Writable Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class WritableTest extends BaseND4JTest {
|
class WritableTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -56,6 +56,12 @@
|
||||||
<artifactId>arrow-format</artifactId>
|
<artifactId>arrow-format</artifactId>
|
||||||
<version>${arrow.version}</version>
|
<version>${arrow.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.nd4j</groupId>
|
||||||
|
<artifactId>nd4j-common-tests</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<profiles>
|
<profiles>
|
||||||
|
|
|
@ -42,9 +42,11 @@ import org.datavec.api.writable.*;
|
||||||
import org.datavec.arrow.recordreader.ArrowRecordReader;
|
import org.datavec.arrow.recordreader.ArrowRecordReader;
|
||||||
import org.datavec.arrow.recordreader.ArrowWritableRecordBatch;
|
import org.datavec.arrow.recordreader.ArrowWritableRecordBatch;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
import org.nd4j.common.primitives.Pair;
|
import org.nd4j.common.primitives.Pair;
|
||||||
|
@ -62,6 +64,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@DisplayName("Arrow Converter Test")
|
@DisplayName("Arrow Converter Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class ArrowConverterTest extends BaseND4JTest {
|
class ArrowConverterTest extends BaseND4JTest {
|
||||||
|
|
||||||
private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE);
|
private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE);
|
||||||
|
@ -142,8 +146,8 @@ class ArrowConverterTest extends BaseND4JTest {
|
||||||
List<FieldVector> fieldVectorsBatch = ArrowConverter.toArrowColumnsString(bufferAllocator, schema.build(), batch);
|
List<FieldVector> fieldVectorsBatch = ArrowConverter.toArrowColumnsString(bufferAllocator, schema.build(), batch);
|
||||||
List<List<Writable>> batchRecords = ArrowConverter.toArrowWritables(fieldVectorsBatch, schema.build());
|
List<List<Writable>> batchRecords = ArrowConverter.toArrowWritables(fieldVectorsBatch, schema.build());
|
||||||
List<List<Writable>> assertionBatch = new ArrayList<>();
|
List<List<Writable>> assertionBatch = new ArrayList<>();
|
||||||
assertionBatch.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(0)));
|
assertionBatch.add(Arrays.asList(new IntWritable(0), new IntWritable(0)));
|
||||||
assertionBatch.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1)));
|
assertionBatch.add(Arrays.asList(new IntWritable(1), new IntWritable(1)));
|
||||||
assertEquals(assertionBatch, batchRecords);
|
assertEquals(assertionBatch, batchRecords);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -156,11 +160,11 @@ class ArrowConverterTest extends BaseND4JTest {
|
||||||
schema.addColumnTime(String.valueOf(i), TimeZone.getDefault());
|
schema.addColumnTime(String.valueOf(i), TimeZone.getDefault());
|
||||||
single.add(String.valueOf(i));
|
single.add(String.valueOf(i));
|
||||||
}
|
}
|
||||||
List<List<Writable>> input = Arrays.asList(Arrays.<Writable>asList(new LongWritable(0), new LongWritable(1)), Arrays.<Writable>asList(new LongWritable(2), new LongWritable(3)));
|
List<List<Writable>> input = Arrays.asList(Arrays.asList(new LongWritable(0), new LongWritable(1)), Arrays.<Writable>asList(new LongWritable(2), new LongWritable(3)));
|
||||||
List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator, schema.build(), input);
|
List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator, schema.build(), input);
|
||||||
ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector, schema.build());
|
ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector, schema.build());
|
||||||
List<Writable> assertion = Arrays.<Writable>asList(new LongWritable(4), new LongWritable(5));
|
List<Writable> assertion = Arrays.asList(new LongWritable(4), new LongWritable(5));
|
||||||
writableRecordBatch.set(1, Arrays.<Writable>asList(new LongWritable(4), new LongWritable(5)));
|
writableRecordBatch.set(1, Arrays.asList(new LongWritable(4), new LongWritable(5)));
|
||||||
List<Writable> recordTest = writableRecordBatch.get(1);
|
List<Writable> recordTest = writableRecordBatch.get(1);
|
||||||
assertEquals(assertion, recordTest);
|
assertEquals(assertion, recordTest);
|
||||||
}
|
}
|
||||||
|
@ -174,11 +178,11 @@ class ArrowConverterTest extends BaseND4JTest {
|
||||||
schema.addColumnInteger(String.valueOf(i));
|
schema.addColumnInteger(String.valueOf(i));
|
||||||
single.add(String.valueOf(i));
|
single.add(String.valueOf(i));
|
||||||
}
|
}
|
||||||
List<List<Writable>> input = Arrays.asList(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(1)), Arrays.<Writable>asList(new IntWritable(2), new IntWritable(3)));
|
List<List<Writable>> input = Arrays.asList(Arrays.asList(new IntWritable(0), new IntWritable(1)), Arrays.<Writable>asList(new IntWritable(2), new IntWritable(3)));
|
||||||
List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator, schema.build(), input);
|
List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator, schema.build(), input);
|
||||||
ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector, schema.build());
|
ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector, schema.build());
|
||||||
List<Writable> assertion = Arrays.<Writable>asList(new IntWritable(4), new IntWritable(5));
|
List<Writable> assertion = Arrays.asList(new IntWritable(4), new IntWritable(5));
|
||||||
writableRecordBatch.set(1, Arrays.<Writable>asList(new IntWritable(4), new IntWritable(5)));
|
writableRecordBatch.set(1, Arrays.asList(new IntWritable(4), new IntWritable(5)));
|
||||||
List<Writable> recordTest = writableRecordBatch.get(1);
|
List<Writable> recordTest = writableRecordBatch.get(1);
|
||||||
assertEquals(assertion, recordTest);
|
assertEquals(assertion, recordTest);
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,6 +33,7 @@ import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.datavec.arrow.recordreader.ArrowRecordReader;
|
import org.datavec.arrow.recordreader.ArrowRecordReader;
|
||||||
import org.datavec.arrow.recordreader.ArrowRecordWriter;
|
import org.datavec.arrow.recordreader.ArrowRecordWriter;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
import org.nd4j.common.primitives.Triple;
|
import org.nd4j.common.primitives.Triple;
|
||||||
|
@ -44,8 +45,11 @@ import java.util.List;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Record Mapper Test")
|
@DisplayName("Record Mapper Test")
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class RecordMapperTest extends BaseND4JTest {
|
class RecordMapperTest extends BaseND4JTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -30,8 +30,10 @@ import org.datavec.api.writable.Text;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.datavec.arrow.ArrowConverter;
|
import org.datavec.arrow.ArrowConverter;
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.tests.BaseND4JTest;
|
import org.nd4j.common.tests.BaseND4JTest;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -39,13 +41,15 @@ import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
|
public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
|
||||||
|
|
||||||
private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE);
|
private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE);
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@Tag(TagNames.NEEDS_VERIFY)
|
||||||
@Disabled
|
@Disabled
|
||||||
public void testBasicIndexing() {
|
public void testBasicIndexing() {
|
||||||
Schema.Builder schema = new Schema.Builder();
|
Schema.Builder schema = new Schema.Builder();
|
||||||
|
@ -79,8 +83,9 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
//not worried about this till after next release
|
@Tag(TagNames.NEEDS_VERIFY)
|
||||||
@Disabled
|
@Disabled
|
||||||
|
//not worried about this till after next release
|
||||||
public void testVariableLengthTS() {
|
public void testVariableLengthTS() {
|
||||||
Schema.Builder schema = new Schema.Builder()
|
Schema.Builder schema = new Schema.Builder()
|
||||||
.addColumnString("str")
|
.addColumnString("str")
|
||||||
|
@ -88,13 +93,13 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest {
|
||||||
.addColumnDouble("dbl");
|
.addColumnDouble("dbl");
|
||||||
|
|
||||||
List<List<Writable>> firstSeq = Arrays.asList(
|
List<List<Writable>> firstSeq = Arrays.asList(
|
||||||
Arrays.<Writable>asList(new Text("00"),new IntWritable(0),new DoubleWritable(2.0)),
|
Arrays.asList(new Text("00"),new IntWritable(0),new DoubleWritable(2.0)),
|
||||||
Arrays.<Writable>asList(new Text("01"),new IntWritable(1),new DoubleWritable(2.1)),
|
Arrays.asList(new Text("01"),new IntWritable(1),new DoubleWritable(2.1)),
|
||||||
Arrays.<Writable>asList(new Text("02"),new IntWritable(2),new DoubleWritable(2.2)));
|
Arrays.asList(new Text("02"),new IntWritable(2),new DoubleWritable(2.2)));
|
||||||
|
|
||||||
List<List<Writable>> secondSeq = Arrays.asList(
|
List<List<Writable>> secondSeq = Arrays.asList(
|
||||||
Arrays.<Writable>asList(new Text("10"),new IntWritable(10),new DoubleWritable(12.0)),
|
Arrays.asList(new Text("10"),new IntWritable(10),new DoubleWritable(12.0)),
|
||||||
Arrays.<Writable>asList(new Text("11"),new IntWritable(11),new DoubleWritable(12.1)));
|
Arrays.asList(new Text("11"),new IntWritable(11),new DoubleWritable(12.1)));
|
||||||
|
|
||||||
List<List<List<Writable>>> sequences = Arrays.asList(firstSeq, secondSeq);
|
List<List<List<Writable>>> sequences = Arrays.asList(firstSeq, secondSeq);
|
||||||
|
|
||||||
|
|
|
@ -100,6 +100,17 @@
|
||||||
<artifactId>hdf5-platform</artifactId>
|
<artifactId>hdf5-platform</artifactId>
|
||||||
<version>${hdf5.version}-${javacpp-presets.version}</version>
|
<version>${hdf5.version}-${javacpp-presets.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.bytedeco</groupId>
|
||||||
|
<artifactId>ffmpeg-platform</artifactId>
|
||||||
|
<version>${ffmpeg.version}-${javacpp.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.nd4j</groupId>
|
||||||
|
<artifactId>nd4j-common-tests</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
|
|
@ -25,6 +25,7 @@ import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.image.recordreader.ImageRecordReader;
|
import org.datavec.image.recordreader.ImageRecordReader;
|
||||||
|
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
@ -35,19 +36,25 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Label Generator Test")
|
@DisplayName("Label Generator Test")
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class LabelGeneratorTest {
|
class LabelGeneratorTest {
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@DisplayName("Test Parent Path Label Generator")
|
@DisplayName("Test Parent Path Label Generator")
|
||||||
@Disabled
|
|
||||||
void testParentPathLabelGenerator(@TempDir Path testDir) throws Exception {
|
void testParentPathLabelGenerator(@TempDir Path testDir) throws Exception {
|
||||||
File orig = new ClassPathResource("datavec-data-image/testimages/class0/0.jpg").getFile();
|
File orig = new ClassPathResource("datavec-data-image/testimages/class0/0.jpg").getFile();
|
||||||
for (String dirPrefix : new String[] { "m.", "m" }) {
|
for (String dirPrefix : new String[] { "m.", "m" }) {
|
||||||
File f = testDir.toFile();
|
File f = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile();
|
||||||
|
f.mkdirs();
|
||||||
int numDirs = 3;
|
int numDirs = 3;
|
||||||
int filesPerDir = 4;
|
int filesPerDir = 4;
|
||||||
for (int i = 0; i < numDirs; i++) {
|
for (int i = 0; i < numDirs; i++) {
|
||||||
|
|
|
@ -23,7 +23,10 @@ package org.datavec.image.loader;
|
||||||
import org.apache.commons.io.FilenameUtils;
|
import org.apache.commons.io.FilenameUtils;
|
||||||
import org.datavec.api.records.reader.RecordReader;
|
import org.datavec.api.records.reader.RecordReader;
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.dataset.DataSet;
|
import org.nd4j.linalg.dataset.DataSet;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
@ -39,6 +42,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class LoaderTests {
|
public class LoaderTests {
|
||||||
|
|
||||||
private static void ensureDataAvailable(){
|
private static void ensureDataAvailable(){
|
||||||
|
@ -81,7 +86,7 @@ public class LoaderTests {
|
||||||
String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin";
|
String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin";
|
||||||
String path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
|
String path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
|
||||||
byte[] fullDataExpected = new byte[3073];
|
byte[] fullDataExpected = new byte[3073];
|
||||||
FileInputStream inExpected = new FileInputStream(new File(path));
|
FileInputStream inExpected = new FileInputStream(path);
|
||||||
inExpected.read(fullDataExpected);
|
inExpected.read(fullDataExpected);
|
||||||
|
|
||||||
byte[] fullDataActual = new byte[3073];
|
byte[] fullDataActual = new byte[3073];
|
||||||
|
@ -94,7 +99,7 @@ public class LoaderTests {
|
||||||
subDir = "cifar/cifar-10-batches-bin/test_batch.bin";
|
subDir = "cifar/cifar-10-batches-bin/test_batch.bin";
|
||||||
path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
|
path = FilenameUtils.concat(System.getProperty("user.home"), subDir);
|
||||||
fullDataExpected = new byte[3073];
|
fullDataExpected = new byte[3073];
|
||||||
inExpected = new FileInputStream(new File(path));
|
inExpected = new FileInputStream(path);
|
||||||
inExpected.read(fullDataExpected);
|
inExpected.read(fullDataExpected);
|
||||||
|
|
||||||
fullDataActual = new byte[3073];
|
fullDataActual = new byte[3073];
|
||||||
|
|
|
@ -21,8 +21,11 @@
|
||||||
package org.datavec.image.loader;
|
package org.datavec.image.loader;
|
||||||
|
|
||||||
import org.datavec.image.data.Image;
|
import org.datavec.image.data.Image;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.resources.Resources;
|
import org.nd4j.common.resources.Resources;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
|
|
||||||
import java.awt.image.BufferedImage;
|
import java.awt.image.BufferedImage;
|
||||||
|
@ -34,7 +37,8 @@ import java.util.Random;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestImageLoader {
|
public class TestImageLoader {
|
||||||
|
|
||||||
private static long seed = 10;
|
private static long seed = 10;
|
||||||
|
|
|
@ -31,10 +31,13 @@ import org.bytedeco.javacv.OpenCVFrameConverter;
|
||||||
import org.datavec.image.data.Image;
|
import org.datavec.image.data.Image;
|
||||||
import org.datavec.image.data.ImageWritable;
|
import org.datavec.image.data.ImageWritable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.resources.Resources;
|
import org.nd4j.common.resources.Resources;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.buffer.DataType;
|
import org.nd4j.linalg.api.buffer.DataType;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
|
@ -60,6 +63,8 @@ import static org.junit.jupiter.api.Assertions.fail;
|
||||||
* @author saudet
|
* @author saudet
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestNativeImageLoader {
|
public class TestNativeImageLoader {
|
||||||
static final long seed = 10;
|
static final long seed = 10;
|
||||||
static final Random rng = new Random(seed);
|
static final Random rng = new Random(seed);
|
||||||
|
|
|
@ -28,9 +28,12 @@ import org.datavec.api.writable.NDArrayWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.datavec.image.loader.NativeImageLoader;
|
import org.datavec.image.loader.NativeImageLoader;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.loader.FileBatch;
|
import org.nd4j.common.loader.FileBatch;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
@ -41,6 +44,8 @@ import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
|
||||||
@DisplayName("File Batch Record Reader Test")
|
@DisplayName("File Batch Record Reader Test")
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class FileBatchRecordReaderTest {
|
class FileBatchRecordReaderTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -37,9 +37,12 @@ import org.datavec.api.writable.NDArrayWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
import org.datavec.api.writable.batch.NDArrayRecordBatch;
|
import org.datavec.api.writable.batch.NDArrayRecordBatch;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.buffer.DataType;
|
import org.nd4j.linalg.api.buffer.DataType;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
|
@ -54,7 +57,8 @@ import java.util.List;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestImageRecordReader {
|
public class TestImageRecordReader {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -36,9 +36,12 @@ import org.datavec.image.transform.ImageTransform;
|
||||||
import org.datavec.image.transform.PipelineImageTransform;
|
import org.datavec.image.transform.PipelineImageTransform;
|
||||||
import org.datavec.image.transform.ResizeImageTransform;
|
import org.datavec.image.transform.ResizeImageTransform;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||||
import org.nd4j.linalg.factory.Nd4j;
|
import org.nd4j.linalg.factory.Nd4j;
|
||||||
import org.nd4j.linalg.indexing.BooleanIndexing;
|
import org.nd4j.linalg.indexing.BooleanIndexing;
|
||||||
|
@ -54,7 +57,8 @@ import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestObjectDetectionRecordReader {
|
public class TestObjectDetectionRecordReader {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -22,10 +22,13 @@ package org.datavec.image.recordreader.objdetect;
|
||||||
|
|
||||||
import org.datavec.image.recordreader.objdetect.impl.VocLabelProvider;
|
import org.datavec.image.recordreader.objdetect.impl.VocLabelProvider;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
@ -34,7 +37,8 @@ import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
public class TestVocLabelProvider {
|
public class TestVocLabelProvider {
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
package org.datavec.image.transform;
|
package org.datavec.image.transform;
|
||||||
|
|
||||||
import org.datavec.image.data.ImageWritable;
|
import org.datavec.image.data.ImageWritable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -28,8 +29,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Json Yaml Test")
|
@DisplayName("Json Yaml Test")
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JACKSON_SERDE)
|
||||||
class JsonYamlTest {
|
class JsonYamlTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -22,12 +22,17 @@ package org.datavec.image.transform;
|
||||||
import org.bytedeco.javacv.Frame;
|
import org.bytedeco.javacv.Frame;
|
||||||
import org.datavec.image.data.ImageWritable;
|
import org.datavec.image.data.ImageWritable;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Resize Image Transform Test")
|
@DisplayName("Resize Image Transform Test")
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
class ResizeImageTransformTest {
|
class ResizeImageTransformTest {
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
|
|
|
@ -24,6 +24,7 @@ import org.bytedeco.javacpp.indexer.UByteIndexer;
|
||||||
import org.bytedeco.javacv.CanvasFrame;
|
import org.bytedeco.javacv.CanvasFrame;
|
||||||
import org.bytedeco.javacv.Frame;
|
import org.bytedeco.javacv.Frame;
|
||||||
import org.bytedeco.javacv.OpenCVFrameConverter;
|
import org.bytedeco.javacv.OpenCVFrameConverter;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
import org.nd4j.common.primitives.Pair;
|
import org.nd4j.common.primitives.Pair;
|
||||||
import org.datavec.image.data.ImageWritable;
|
import org.datavec.image.data.ImageWritable;
|
||||||
|
@ -37,6 +38,8 @@ import java.util.List;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
|
||||||
import org.bytedeco.opencv.opencv_core.*;
|
import org.bytedeco.opencv.opencv_core.*;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
import static org.bytedeco.opencv.global.opencv_core.*;
|
import static org.bytedeco.opencv.global.opencv_core.*;
|
||||||
import static org.bytedeco.opencv.global.opencv_imgproc.*;
|
import static org.bytedeco.opencv.global.opencv_imgproc.*;
|
||||||
|
@ -46,6 +49,8 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||||
*
|
*
|
||||||
* @author saudet
|
* @author saudet
|
||||||
*/
|
*/
|
||||||
|
@NativeTag
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
public class TestImageTransform {
|
public class TestImageTransform {
|
||||||
static final long seed = 10;
|
static final long seed = 10;
|
||||||
static final Random rng = new Random(seed);
|
static final Random rng = new Random(seed);
|
||||||
|
@ -255,7 +260,6 @@ public class TestImageTransform {
|
||||||
assertEquals(22, transformed[1], 0);
|
assertEquals(22, transformed[1], 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Disabled
|
|
||||||
@Test
|
@Test
|
||||||
public void testFilterImageTransform() throws Exception {
|
public void testFilterImageTransform() throws Exception {
|
||||||
ImageWritable writable = makeRandomImage(0, 0, 4);
|
ImageWritable writable = makeRandomImage(0, 0, 4);
|
||||||
|
|
|
@ -53,6 +53,12 @@
|
||||||
<artifactId>poi-ooxml</artifactId>
|
<artifactId>poi-ooxml</artifactId>
|
||||||
<version>${poi.version}</version>
|
<version>${poi.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.nd4j</groupId>
|
||||||
|
<artifactId>nd4j-common-tests</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<profiles>
|
<profiles>
|
||||||
|
|
|
@ -181,7 +181,7 @@ public class ExcelRecordReader extends FileRecordReader {
|
||||||
List<Writable> ret = new ArrayList<>(currRow.getLastCellNum());
|
List<Writable> ret = new ArrayList<>(currRow.getLastCellNum());
|
||||||
for(Cell cell: currRow) {
|
for(Cell cell: currRow) {
|
||||||
String cellValue = dataFormatter.formatCellValue(cell);
|
String cellValue = dataFormatter.formatCellValue(cell);
|
||||||
switch(cell.getCellTypeEnum()) {
|
switch(cell.getCellType()) {
|
||||||
case BLANK: ret.add(new Text("")); break;
|
case BLANK: ret.add(new Text("")); break;
|
||||||
case STRING: ret.add(new Text("")); break;
|
case STRING: ret.add(new Text("")); break;
|
||||||
case BOOLEAN: ret.add(new BooleanWritable(Boolean.valueOf(cellValue))); break;
|
case BOOLEAN: ret.add(new BooleanWritable(Boolean.valueOf(cellValue))); break;
|
||||||
|
|
|
@ -22,6 +22,7 @@ package org.datavec.poi.excel;
|
||||||
import org.datavec.api.records.reader.RecordReader;
|
import org.datavec.api.records.reader.RecordReader;
|
||||||
import org.datavec.api.split.FileSplit;
|
import org.datavec.api.split.FileSplit;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.nd4j.common.io.ClassPathResource;
|
import org.nd4j.common.io.ClassPathResource;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -29,8 +30,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.NativeTag;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Excel Record Reader Test")
|
@DisplayName("Excel Record Reader Test")
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
class ExcelRecordReaderTest {
|
class ExcelRecordReaderTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.datavec.api.transform.schema.Schema;
|
||||||
import org.datavec.api.writable.IntWritable;
|
import org.datavec.api.writable.IntWritable;
|
||||||
import org.datavec.api.writable.Writable;
|
import org.datavec.api.writable.Writable;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Tag;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.nd4j.common.primitives.Triple;
|
import org.nd4j.common.primitives.Triple;
|
||||||
|
@ -36,8 +37,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.nd4j.common.tests.tags.TagNames;
|
||||||
|
|
||||||
@DisplayName("Excel Record Writer Test")
|
@DisplayName("Excel Record Writer Test")
|
||||||
|
@Tag(TagNames.FILE_IO)
|
||||||
|
@Tag(TagNames.JAVA_ONLY)
|
||||||
class ExcelRecordWriterTest {
|
class ExcelRecordWriterTest {
|
||||||
|
|
||||||
@TempDir
|
@TempDir
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
|
||||||
|
# *************************************************************************
|
||||||
|
# *** DO NOT TOUCH FILES IN THIS DIRECTORY! ***
|
||||||
|
# *** FILES IN THIS DIRECTORY AND SUBDIRECTORIES CONSTITUTE A DERBY ***
|
||||||
|
# *** DATABASE, WHICH INCLUDES THE DATA (USER AND SYSTEM) AND THE ***
|
||||||
|
# *** FILES NECESSARY FOR DATABASE RECOVERY. ***
|
||||||
|
# *** EDITING, ADDING, OR DELETING ANY OF THESE FILES MAY CAUSE DATA ***
|
||||||
|
# *** CORRUPTION AND LEAVE THE DATABASE IN A NON-RECOVERABLE STATE. ***
|
||||||
|
# *************************************************************************
|
Binary file not shown.
|
@ -0,0 +1,8 @@
|
||||||
|
|
||||||
|
# *************************************************************************
|
||||||
|
# *** DO NOT TOUCH FILES IN THIS DIRECTORY! ***
|
||||||
|
# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE RECOVERY ***
|
||||||
|
# *** SYSTEM. EDITING, ADDING, OR DELETING FILES IN THIS DIRECTORY ***
|
||||||
|
# *** WILL CAUSE THE DERBY RECOVERY SYSTEM TO FAIL, LEADING TO ***
|
||||||
|
# *** NON-RECOVERABLE CORRUPT DATABASES. ***
|
||||||
|
# *************************************************************************
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,8 @@
|
||||||
|
|
||||||
|
# *************************************************************************
|
||||||
|
# *** DO NOT TOUCH FILES IN THIS DIRECTORY! ***
|
||||||
|
# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE TO STORE ***
|
||||||
|
# *** USER AND SYSTEM DATA. EDITING, ADDING, OR DELETING FILES IN THIS ***
|
||||||
|
# *** DIRECTORY WILL CORRUPT THE ASSOCIATED DERBY DATABASE AND MAKE ***
|
||||||
|
# *** IT NON-RECOVERABLE. ***
|
||||||
|
# *************************************************************************
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue