arrays = new ArrayList<>();
- long[] vShape = new long[]{1, -1};
- while (iter.hasNext()) {
- if (line.isEmpty())
- line = iter.nextLine();
- String[] split = line.split(" ");
- String word = ReadHelper.decodeB64(split[0]); //split[0].replaceAll(whitespaceReplacement, " ");
- VocabWord word1 = new VocabWord(1.0, word);
+ /* We should check for something that looks like proper word vectors here. i.e: 1 word at the 0
+ * position, and bunch of floats further */
+ String[] headers = line.split(" ");
- word1.setIndex(cache.numWords());
+ try {
+ long[] header = new long[headers.length];
+ for (int x = 0; x < headers.length; x++) {
+ header[x] = Long.parseLong(headers[x]);
+ }
- cache.addToken(word1);
+ /* Now we know, if that's all ints - it's just a header
+ * [0] - number of words
+ * [1] - vectorLength
+ * [2] - number of documents <-- DL4j-only value
+ */
+ if (headers.length == 3) {
+ long numberOfDocuments = header[2];
+ cache.incrementTotalDocCount(numberOfDocuments);
+ }
- cache.addWordToIndex(word1.getIndex(), word);
+ long numWords = header[0];
+ int vectorLength = (int) header[1];
+ printOutProjectedMemoryUse(numWords, vectorLength, 1);
- cache.putVocabWord(word);
-
- float[] vector = new float[split.length - 1];
-
- for (int i = 1; i < split.length; i++) {
- vector[i - 1] = Float.parseFloat(split[i]);
+ return true;
+ } catch (Exception notHeaderException) {
+ // if any conversion exception hits - that'll be considered header
+ return false;
}
-
- vShape[1] = vector.length;
- INDArray row = Nd4j.create(vector, vShape);
-
- arrays.add(row);
-
- // workaround for skipped first row
- line = "";
}
-
- INDArray syn = Nd4j.vstack(arrays);
-
- InMemoryLookupTable lookupTable =
- (InMemoryLookupTable) new InMemoryLookupTable.Builder().vectorLength(arrays.get(0).columns())
- .useAdaGrad(false).cache(cache).useHierarchicSoftmax(false).build();
-
- lookupTable.setSyn0(syn);
-
- iter.close();
-
- try {
- reader.close();
- } catch (Exception e) {
- }
-
- return new Pair<>(lookupTable, (VocabCache) cache);
}
/**
@@ -2352,22 +2411,6 @@ public class WordVectorSerializer {
}
}
- /**
- * This method
- * 1) Binary model, either compressed or not. Like well-known Google Model
- * 2) Popular CSV word2vec text format
- * 3) DL4j compressed format
- *
- * Please note: Only weights will be loaded by this method.
- *
- * @param file
- * @return
- */
- public static Word2Vec readWord2VecModel(@NonNull File file) {
- return readWord2VecModel(file, false);
- }
-
-
/**
* This method
* 1) Binary model, either compressed or not. Like well-known Google Model
@@ -2389,106 +2432,196 @@ public class WordVectorSerializer {
* 2) Popular CSV word2vec text format
* 3) DL4j compressed format
*
- * Please note: if extended data isn't available, only weights will be loaded instead.
+ * Please note: Only weights will be loaded by this method.
*
- * @param path
- * @param extendedModel if TRUE, we'll try to load HS states & Huffman tree info, if FALSE, only weights will be loaded
+ * @param path path to model file
+ * @param extendedModel if TRUE, we'll try to load HS states & Huffman tree info, if FALSE, only weights will be loaded
* @return
*/
public static Word2Vec readWord2VecModel(String path, boolean extendedModel) {
return readWord2VecModel(new File(path), extendedModel);
}
- public static Word2Vec readAsBinaryNoLineBreaks(@NonNull File file) {
+ /**
+ * This method
+ * 1) Binary model, either compressed or not. Like well-known Google Model
+ * 2) Popular CSV word2vec text format
+ * 3) DL4j compressed format
+ *
+ * Please note: Only weights will be loaded by this method.
+ *
+ * @param file
+ * @return
+ */
+ public static Word2Vec readWord2VecModel(File file) {
+ return readWord2VecModel(file, false);
+ }
+
+ /**
+ * This method
+ * 1) Binary model, either compressed or not. Like well-known Google Model
+ * 2) Popular CSV word2vec text format
+ * 3) DL4j compressed format
+ *
+ * Please note: if extended data isn't available, only weights will be loaded instead.
+ *
+ * @param file model file
+ * @param extendedModel if TRUE, we'll try to load HS states & Huffman tree info, if FALSE, only weights will be loaded
+ * @return word2vec model
+ */
+ public static Word2Vec readWord2VecModel(File file, boolean extendedModel) {
+ if (!file.exists() || !file.isFile()) {
+ throw new ND4JIllegalStateException("File [" + file.getAbsolutePath() + "] doesn't exist");
+ }
+ boolean originalPeriodic = Nd4j.getMemoryManager().isPeriodicGcActive();
+ if (originalPeriodic) {
+ Nd4j.getMemoryManager().togglePeriodicGc(false);
+ }
+ Nd4j.getMemoryManager().setOccasionalGcFrequency(50000);
+
+ try {
+ return readWord2Vec(file, extendedModel);
+ } catch (Exception readSequenceVectors) {
+ try {
+ return extendedModel
+ ? readAsExtendedModel(file)
+ : readAsSimplifiedModel(file);
+ } catch (Exception loadFromFileException) {
+ try {
+ return readAsCsv(file);
+ } catch (Exception readCsvException) {
+ try {
+ return readAsBinary(file);
+ } catch (Exception readBinaryException) {
+ try {
+ return readAsBinaryNoLineBreaks(file);
+ } catch (Exception readModelException) {
+ log.error("Unable to guess input file format", readModelException);
+ throw new RuntimeException("Unable to guess input file format. Please use corresponding loader directly");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public static Word2Vec readAsBinaryNoLineBreaks(@NonNull File file) {
+ try (InputStream inputStream = fileStream(file)) {
+ return readAsBinaryNoLineBreaks(inputStream);
+ } catch (IOException readCsvException) {
+ throw new RuntimeException(readCsvException);
+ }
+ }
+
+ public static Word2Vec readAsBinaryNoLineBreaks(@NonNull InputStream inputStream) {
boolean originalPeriodic = Nd4j.getMemoryManager().isPeriodicGcActive();
int originalFreq = Nd4j.getMemoryManager().getOccasionalGcFrequency();
- Word2Vec vec;
// try to load without linebreaks
try {
- if (originalPeriodic)
+ if (originalPeriodic) {
Nd4j.getMemoryManager().togglePeriodicGc(true);
+ }
Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);
- vec = readBinaryModel(file, false, false);
- return vec;
- } catch (Exception ez) {
- throw new RuntimeException(
- "Unable to guess input file format. Please use corresponding loader directly");
+ return readBinaryModel(inputStream, false, false);
+ } catch (Exception readModelException) {
+ log.error("Cannot read binary model", readModelException);
+ throw new RuntimeException("Unable to guess input file format. Please use corresponding loader directly");
+ }
+ }
+
+ public static Word2Vec readAsBinary(@NonNull File file) {
+ try (InputStream inputStream = fileStream(file)) {
+ return readAsBinary(inputStream);
+ } catch (IOException readCsvException) {
+ throw new RuntimeException(readCsvException);
}
}
/**
- * This method loads Word2Vec model from binary file
+ * This method loads Word2Vec model from binary input stream.
*
- * @param file File
- * @return Word2Vec
+ * @param inputStream binary input stream
+ * @return Word2Vec
*/
- public static Word2Vec readAsBinary(@NonNull File file) {
+ public static Word2Vec readAsBinary(@NonNull InputStream inputStream) {
boolean originalPeriodic = Nd4j.getMemoryManager().isPeriodicGcActive();
int originalFreq = Nd4j.getMemoryManager().getOccasionalGcFrequency();
- Word2Vec vec;
-
// we fallback to trying binary model instead
try {
log.debug("Trying binary model restoration...");
- if (originalPeriodic)
+ if (originalPeriodic) {
Nd4j.getMemoryManager().togglePeriodicGc(true);
+ }
Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);
- vec = readBinaryModel(file, true, false);
- return vec;
- } catch (Exception ey) {
- throw new RuntimeException(ey);
+ return readBinaryModel(inputStream, true, false);
+ } catch (Exception readModelException) {
+ throw new RuntimeException(readModelException);
+ }
+ }
+
+ public static Word2Vec readAsCsv(@NonNull File file) {
+ try (InputStream inputStream = fileStream(file)) {
+ return readAsCsv(inputStream);
+ } catch (IOException readCsvException) {
+ throw new RuntimeException(readCsvException);
}
}
/**
* This method loads Word2Vec model from csv file
*
- * @param file File
- * @return Word2Vec
+ * @param inputStream input stream
+ * @return Word2Vec model
*/
- public static Word2Vec readAsCsv(@NonNull File file) {
-
- Word2Vec vec;
+ public static Word2Vec readAsCsv(@NonNull InputStream inputStream) {
VectorsConfiguration configuration = new VectorsConfiguration();
// let's try to load this file as csv file
try {
log.debug("Trying CSV model restoration...");
- Pair pair = loadTxt(file);
- Word2Vec.Builder builder = new Word2Vec.Builder().lookupTable(pair.getFirst()).useAdaGrad(false)
- .vocabCache(pair.getSecond()).layerSize(pair.getFirst().layerSize())
+ Pair pair = loadTxt(inputStream);
+ Word2Vec.Builder builder = new Word2Vec
+ .Builder()
+ .lookupTable(pair.getFirst())
+ .useAdaGrad(false)
+ .vocabCache(pair.getSecond())
+ .layerSize(pair.getFirst().layerSize())
// we don't use hs here, because model is incomplete
- .useHierarchicSoftmax(false).resetModel(false);
+ .useHierarchicSoftmax(false)
+ .resetModel(false);
TokenizerFactory factory = getTokenizerFactory(configuration);
- if (factory != null)
+ if (factory != null) {
builder.tokenizerFactory(factory);
+ }
- vec = builder.build();
- return vec;
+ return builder.build();
} catch (Exception ex) {
throw new RuntimeException("Unable to load model in CSV format");
}
}
+ /**
+ * This method just loads full compressed model.
+ */
private static Word2Vec readAsExtendedModel(@NonNull File file) throws IOException {
int originalFreq = Nd4j.getMemoryManager().getOccasionalGcFrequency();
boolean originalPeriodic = Nd4j.getMemoryManager().isPeriodicGcActive();
log.debug("Trying full model restoration...");
- // this method just loads full compressed model
- if (originalPeriodic)
+ if (originalPeriodic) {
Nd4j.getMemoryManager().togglePeriodicGc(true);
+ }
Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);
@@ -2627,67 +2760,6 @@ public class WordVectorSerializer {
return vec;
}
- /**
- * This method
- * 1) Binary model, either compressed or not. Like well-known Google Model
- * 2) Popular CSV word2vec text format
- * 3) DL4j compressed format
- *
- * Please note: if extended data isn't available, only weights will be loaded instead.
- *
- * @param file
- * @param extendedModel if TRUE, we'll try to load HS states & Huffman tree info, if FALSE, only weights will be loaded
- * @return
- */
- public static Word2Vec readWord2VecModel(@NonNull File file, boolean extendedModel) {
-
- if (!file.exists() || !file.isFile())
- throw new ND4JIllegalStateException("File [" + file.getAbsolutePath() + "] doesn't exist");
-
- Word2Vec vec = null;
-
- int originalFreq = Nd4j.getMemoryManager().getOccasionalGcFrequency();
- boolean originalPeriodic = Nd4j.getMemoryManager().isPeriodicGcActive();
- if (originalPeriodic)
- Nd4j.getMemoryManager().togglePeriodicGc(false);
- Nd4j.getMemoryManager().setOccasionalGcFrequency(50000);
-
- // try to load zip format
- try {
- vec = readWord2Vec(file, extendedModel);
- return vec;
- } catch (Exception e) {
- // let's try to load this file as csv file
- try {
- if (extendedModel) {
- vec = readAsExtendedModel(file);
- return vec;
- } else {
- vec = readAsSimplifiedModel(file);
- return vec;
- }
- } catch (Exception ex) {
- try {
- vec = readAsCsv(file);
- return vec;
- } catch (Exception exc) {
- try {
- vec = readAsBinary(file);
- return vec;
- } catch (Exception exce) {
- try {
- vec = readAsBinaryNoLineBreaks(file);
- return vec;
-
- } catch (Exception excep) {
- throw new RuntimeException("Unable to guess input file format. Please use corresponding loader directly");
- }
- }
- }
- }
- }
- }
-
protected static TokenizerFactory getTokenizerFactory(VectorsConfiguration configuration) {
if (configuration == null)
return null;
@@ -3019,16 +3091,13 @@ public class WordVectorSerializer {
/**
* This method restores Word2Vec model from file
*
- * @param path String
- * @param readExtendedTables booleab
+ * @param path
+ * @param readExtendedTables
* @return Word2Vec
*/
- public static Word2Vec readWord2Vec(@NonNull String path, boolean readExtendedTables)
- throws IOException {
-
+ public static Word2Vec readWord2Vec(@NonNull String path, boolean readExtendedTables) {
File file = new File(path);
- Word2Vec word2Vec = readWord2Vec(file, readExtendedTables);
- return word2Vec;
+ return readWord2Vec(file, readExtendedTables);
}
/**
@@ -3139,11 +3208,12 @@ public class WordVectorSerializer {
* @param readExtendedTables boolean
* @return Word2Vec
*/
- public static Word2Vec readWord2Vec(@NonNull File file, boolean readExtendedTables)
- throws IOException {
-
- Word2Vec word2Vec = readWord2Vec(new FileInputStream(file), readExtendedTables);
- return word2Vec;
+ public static Word2Vec readWord2Vec(@NonNull File file, boolean readExtendedTables) {
+ try (InputStream inputStream = fileStream(file)) {
+ return readWord2Vec(inputStream, readExtendedTables);
+ } catch (Exception readSequenceVectors) {
+ throw new RuntimeException(readSequenceVectors);
+ }
}
/**
@@ -3153,13 +3223,19 @@ public class WordVectorSerializer {
* @param readExtendedTable boolean
* @return Word2Vec
*/
- public static Word2Vec readWord2Vec(@NonNull InputStream stream,
- boolean readExtendedTable) throws IOException {
+ public static Word2Vec readWord2Vec(
+ @NonNull InputStream stream,
+ boolean readExtendedTable) throws IOException {
SequenceVectors vectors = readSequenceVectors(stream, readExtendedTable);
- Word2Vec word2Vec = new Word2Vec.Builder(vectors.getConfiguration()).layerSize(vectors.getLayerSize()).build();
+
+ Word2Vec word2Vec = new Word2Vec
+ .Builder(vectors.getConfiguration())
+ .layerSize(vectors.getLayerSize())
+ .build();
word2Vec.setVocab(vectors.getVocab());
word2Vec.setLookupTable(vectors.lookupTable());
word2Vec.setModelUtils(vectors.getModelUtils());
+
return word2Vec;
}
diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/TsneTest.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/TsneTest.java
index 69fcd236c..5466bc15b 100644
--- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/TsneTest.java
+++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/TsneTest.java
@@ -37,8 +37,6 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
-import static org.junit.Assert.assertEquals;
-
@Slf4j
public class TsneTest extends BaseDL4JTest {
diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/WordVectorSerializerTest.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java
similarity index 86%
rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/WordVectorSerializerTest.java
rename to deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java
index b7aff923e..f089a6ae9 100644
--- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/WordVectorSerializerTest.java
+++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java
@@ -14,17 +14,14 @@
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
-package org.deeplearning4j.models.sequencevectors.serialization;
+package org.deeplearning4j.models.embeddings.loader;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.models.embeddings.WeightLookupTable;
import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
import org.deeplearning4j.models.embeddings.learning.impl.elements.CBOW;
-import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration;
-import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
import org.deeplearning4j.models.embeddings.reader.impl.BasicModelUtils;
import org.deeplearning4j.models.embeddings.reader.impl.FlatModelUtils;
import org.deeplearning4j.models.fasttext.FastText;
@@ -47,7 +44,11 @@ import java.io.File;
import java.io.IOException;
import java.util.Collections;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
@Slf4j
public class WordVectorSerializerTest extends BaseDL4JTest {
@@ -78,10 +79,11 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
syn1 = Nd4j.rand(DataType.FLOAT, 10, 2),
syn1Neg = Nd4j.rand(DataType.FLOAT, 10, 2);
- InMemoryLookupTable lookupTable =
- (InMemoryLookupTable) new InMemoryLookupTable.Builder()
- .useAdaGrad(false).cache(cache)
- .build();
+ InMemoryLookupTable lookupTable = new InMemoryLookupTable
+ .Builder()
+ .useAdaGrad(false)
+ .cache(cache)
+ .build();
lookupTable.setSyn0(syn0);
lookupTable.setSyn1(syn1);
@@ -92,7 +94,6 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
lookupTable(lookupTable).
build();
SequenceVectors deser = null;
- String json = StringUtils.EMPTY;
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
WordVectorSerializer.writeSequenceVectors(vectors, baos);
@@ -126,10 +127,11 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
syn1 = Nd4j.rand(DataType.FLOAT, 10, 2),
syn1Neg = Nd4j.rand(DataType.FLOAT, 10, 2);
- InMemoryLookupTable lookupTable =
- (InMemoryLookupTable) new InMemoryLookupTable.Builder()
- .useAdaGrad(false).cache(cache)
- .build();
+ InMemoryLookupTable lookupTable = new InMemoryLookupTable
+ .Builder()
+ .useAdaGrad(false)
+ .cache(cache)
+ .build();
lookupTable.setSyn0(syn0);
lookupTable.setSyn1(syn1);
@@ -204,10 +206,11 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
syn1 = Nd4j.rand(DataType.FLOAT, 10, 2),
syn1Neg = Nd4j.rand(DataType.FLOAT, 10, 2);
- InMemoryLookupTable lookupTable =
- (InMemoryLookupTable) new InMemoryLookupTable.Builder()
- .useAdaGrad(false).cache(cache)
- .build();
+ InMemoryLookupTable lookupTable = new InMemoryLookupTable
+ .Builder()
+ .useAdaGrad(false)
+ .cache(cache)
+ .build();
lookupTable.setSyn0(syn0);
lookupTable.setSyn1(syn1);
@@ -252,10 +255,11 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
syn1 = Nd4j.rand(DataType.FLOAT, 10, 2),
syn1Neg = Nd4j.rand(DataType.FLOAT, 10, 2);
- InMemoryLookupTable lookupTable =
- (InMemoryLookupTable) new InMemoryLookupTable.Builder()
- .useAdaGrad(false).cache(cache)
- .build();
+ InMemoryLookupTable lookupTable = new InMemoryLookupTable
+ .Builder()
+ .useAdaGrad(false)
+ .cache(cache)
+ .build();
lookupTable.setSyn0(syn0);
lookupTable.setSyn1(syn1);
@@ -267,7 +271,6 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
WeightLookupTable deser = null;
try {
WordVectorSerializer.writeLookupTable(lookupTable, file);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
deser = WordVectorSerializer.readLookupTable(file);
} catch (Exception e) {
log.error("",e);
@@ -305,7 +308,6 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
FastText deser = null;
try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
deser = WordVectorSerializer.readWordVectors(new File(dir, "some.data"));
} catch (Exception e) {
log.error("",e);
@@ -323,4 +325,32 @@ public class WordVectorSerializerTest extends BaseDL4JTest {
assertEquals(fastText.getInputFile(), deser.getInputFile());
assertEquals(fastText.getOutputFile(), deser.getOutputFile());
}
+
+ @Test
+ public void testIsHeader_withValidHeader () {
+
+ /* Given */
+ AbstractCache cache = new AbstractCache<>();
+ String line = "48 100";
+
+ /* When */
+ boolean isHeader = WordVectorSerializer.isHeader(line, cache);
+
+ /* Then */
+ assertTrue(isHeader);
+ }
+
+ @Test
+ public void testIsHeader_notHeader () {
+
+ /* Given */
+ AbstractCache cache = new AbstractCache<>();
+ String line = "your -0.0017603 0.0030831 0.00069072 0.0020581 -0.0050952 -2.2573e-05 -0.001141";
+
+ /* When */
+ boolean isHeader = WordVectorSerializer.isHeader(line, cache);
+
+ /* Then */
+ assertFalse(isHeader);
+ }
}
diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java
index 4f0548ef5..5000175d4 100644
--- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java
+++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java
@@ -1,9 +1,9 @@
package org.deeplearning4j.models.fasttext;
import lombok.extern.slf4j.Slf4j;
+import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
import org.deeplearning4j.models.word2vec.Word2Vec;
-import org.deeplearning4j.BaseDL4JTest;
import org.deeplearning4j.text.sentenceiterator.BasicLineIterator;
import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
import org.junit.Rule;
@@ -14,13 +14,14 @@ import org.nd4j.common.primitives.Pair;
import org.nd4j.common.resources.Resources;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
-
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-
@Slf4j
public class FastTextTest extends BaseDL4JTest {
@@ -32,7 +33,6 @@ public class FastTextTest extends BaseDL4JTest {
private File cbowModelFile = Resources.asFile("models/fasttext/cbow.model.bin");
private File supervisedVectors = Resources.asFile("models/fasttext/supervised.model.vec");
-
@Rule
public TemporaryFolder testDir = new TemporaryFolder();
@@ -90,7 +90,7 @@ public class FastTextTest extends BaseDL4JTest {
}
@Test
- public void tesLoadCBOWModel() throws IOException {
+ public void tesLoadCBOWModel() {
FastText fastText = new FastText(cbowModelFile);
fastText.test(cbowModelFile);
@@ -155,7 +155,7 @@ public class FastTextTest extends BaseDL4JTest {
}
@Test
- public void testVocabulary() throws IOException {
+ public void testVocabulary() {
FastText fastText = new FastText(supModelFile);
assertEquals(48, fastText.vocab().numWords());
assertEquals(48, fastText.vocabSize());
@@ -171,78 +171,73 @@ public class FastTextTest extends BaseDL4JTest {
}
@Test
- public void testLoadIterator() {
- try {
- SentenceIterator iter = new BasicLineIterator(inputFile.getAbsolutePath());
- FastText fastText =
- FastText.builder().supervised(true).iterator(iter).build();
- fastText.loadIterator();
-
- } catch (IOException e) {
- log.error("",e);
- }
+ public void testLoadIterator() throws FileNotFoundException {
+ SentenceIterator iter = new BasicLineIterator(inputFile.getAbsolutePath());
+ FastText
+ .builder()
+ .supervised(true)
+ .iterator(iter)
+ .build()
+ .loadIterator();
}
@Test(expected=IllegalStateException.class)
public void testState() {
FastText fastText = new FastText();
- String label = fastText.predict("something");
+ fastText.predict("something");
}
@Test
public void testPretrainedVectors() throws IOException {
File output = testDir.newFile();
- FastText fastText =
- FastText.builder().supervised(true).
- inputFile(inputFile.getAbsolutePath()).
- pretrainedVectorsFile(supervisedVectors.getAbsolutePath()).
- outputFile(output.getAbsolutePath()).build();
+ FastText fastText = FastText
+ .builder()
+ .supervised(true)
+ .inputFile(inputFile.getAbsolutePath())
+ .pretrainedVectorsFile(supervisedVectors.getAbsolutePath())
+ .outputFile(output.getAbsolutePath())
+ .build();
+
log.info("\nTraining supervised model ...\n");
fastText.fit();
}
@Test
public void testWordsStatistics() throws IOException {
-
File output = testDir.newFile();
- FastText fastText =
- FastText.builder().supervised(true).
- inputFile(inputFile.getAbsolutePath()).
- outputFile(output.getAbsolutePath()).build();
+ FastText fastText = FastText
+ .builder()
+ .supervised(true)
+ .inputFile(inputFile.getAbsolutePath())
+ .outputFile(output.getAbsolutePath())
+ .build();
log.info("\nTraining supervised model ...\n");
fastText.fit();
- Word2Vec word2Vec = WordVectorSerializer.readAsCsv(new File(output.getAbsolutePath() + ".vec"));
+ File file = new File(output.getAbsolutePath() + ".vec");
+ Word2Vec word2Vec = WordVectorSerializer.readAsCsv(file);
- assertEquals(48, word2Vec.getVocab().numWords());
-
- System.out.println(word2Vec.wordsNearest("association", 3));
- System.out.println(word2Vec.similarity("Football", "teams"));
- System.out.println(word2Vec.similarity("professional", "minutes"));
- System.out.println(word2Vec.similarity("java","cpp"));
+ assertEquals(48, word2Vec.getVocab().numWords());
+ assertEquals("", 0.1667751520872116, word2Vec.similarity("Football", "teams"), 1e-4);
+ assertEquals("", 0.10083991289138794, word2Vec.similarity("professional", "minutes"), 1e-4);
+ assertEquals("", Double.NaN, word2Vec.similarity("java","cpp"), 0.0);
+ assertThat(word2Vec.wordsNearest("association", 3), hasItems("Football", "Soccer", "men's"));
}
-
@Test
- public void testWordsNativeStatistics() throws IOException {
-
- File output = testDir.newFile();
-
+ public void testWordsNativeStatistics() {
FastText fastText = new FastText();
fastText.loadPretrainedVectors(supervisedVectors);
log.info("\nTraining supervised model ...\n");
assertEquals(48, fastText.vocab().numWords());
-
- String[] result = new String[3];
- fastText.wordsNearest("association", 3).toArray(result);
- assertArrayEquals(new String[]{"most","eleven","hours"}, result);
+ assertThat(fastText.wordsNearest("association", 3), hasItems("most","eleven","hours"));
assertEquals(0.1657, fastText.similarity("Football", "teams"), 1e-4);
assertEquals(0.3661, fastText.similarity("professional", "minutes"), 1e-4);
- assertEquals(Double.NaN, fastText.similarity("java","cpp"), 1e-4);
+ assertEquals(Double.NaN, fastText.similarity("java","cpp"), 0.0);
}
}