Fix javadoc and cleanup

Signed-off-by: brian <brian@brutex.de>
master
Brian Rosenberger 2022-10-21 16:36:56 +02:00
parent 07c052d822
commit a32644bdb7
18 changed files with 322 additions and 254 deletions

View File

@ -55,6 +55,7 @@ configurations.all {
}
allprojects { Project proj ->
apply plugin: 'com.google.osdetector'
@ -161,3 +162,21 @@ allprojects { Project proj ->
}
}
}
task aggregatedJavadocs(type: Javadoc, description: 'Generate javadocs from all child projects as if it was a single project', group: 'Documentation') {
subprojects.each { proj ->
proj.tasks.withType(Javadoc).each { javadocTask ->
logger.quiet("Adding javadoc for project " + proj.name)
source += javadocTask.source
classpath += javadocTask.classpath
excludes += javadocTask.excludes
includes += javadocTask.includes
}
}
destinationDir = file("$buildDir/docs/javadoc")
title = "$project.name $version API"
options.author true
options.links 'http://docs.oracle.com/javase/8/docs/api/'
options.addStringOption('Xdoclint:none', '-quiet')
}

View File

@ -106,7 +106,7 @@ public class Text extends BinaryComparable implements WritableComparable<BinaryC
if (position < 0)
return -1; // duh.
ByteBuffer bb = ByteBuffer.wrap(bytes).position(position);
ByteBuffer bb = (ByteBuffer) ByteBuffer.wrap(bytes).position(position);
return bytesToCodePoint(bb.slice());
}

View File

@ -1462,7 +1462,7 @@ public class SameDiff extends SDBaseOps {
* A special case of {@link #fit()}.
*
* @param iter The iterator to train the SameDiff instance with
* @param numEpochs The number of epochs for training. Must be > 0
* @param numEpochs The number of epochs for training. Must be &gt; 0
* @param validationIter The DataSetIterator to use for validation (null to skip validation)
* @param validationFrequency The frequency with which to run validation. 1 is every epoch, 2 is every other, etc.
* @param listeners Additional listeners to use during this operation
@ -1479,7 +1479,7 @@ public class SameDiff extends SDBaseOps {
* A special case of {@link #fit()}.
*
* @param iter The iterator to train the SameDiff instance with
* @param numEpochs The number of epochs for training. Must be > 0
* @param numEpochs The number of epochs for training. Must be &gt; 0
* @param listeners Additional listeners to use during this operation
* @return a {@link History} object containing the history information for this training operation
* (evaluations specified in the {@link TrainingConfig}, loss values, and timing information).
@ -1497,7 +1497,7 @@ public class SameDiff extends SDBaseOps {
* A special case of {@link #fit()}.
*
* @param iter The iterator to train the SameDiff instance with
* @param numEpochs The number of epochs for training. Must be > 0
* @param numEpochs The number of epochs for training. Must be &gt; 0
* @param validationIter The MultiDataSetIterator to use for validation (null to skip validation)
* @param validationFrequency The frequency with which to run validation. 1 is every epoch, 2 is every other, etc.
* @param listeners Additional listeners to use during this operation
@ -1514,7 +1514,7 @@ public class SameDiff extends SDBaseOps {
* A special case of {@link #fit()}.
*
* @param iter The iterator to train the SameDiff instance with
* @param numEpochs The number of epochs for training. Must be > 0
* @param numEpochs The number of epochs for training. Must be &gt; 0
* @param listeners Additional listeners to use during this operation
* @return a {@link History} object containing the history information for this training operation
* (evaluations specified in the {@link TrainingConfig}, loss values, and timing information).
@ -3036,7 +3036,6 @@ public class SameDiff extends SDBaseOps {
* See also: {@link VariableType}
*
* @param variables Variables to convert to constants
* @return The (now constant) SDVariables
*/
public void convertToConstants(List<SDVariable> variables) {
if (variables.size() == 0)
@ -3201,7 +3200,7 @@ public class SameDiff extends SDBaseOps {
* For example, {@code z(float) = x(float)+y(float)}, changing both x and y to double results in {@code z(double) = x(double)+y(double)}
* without doing anything to change z's datatype directly (z datatype is inferred from x + y + add op).<br>
* ARRAY type SDVariables cannot be converted directly, as their datatypes are determined by the function +
* input datatypes.<b>
* input datatypes.<br>
* Note that this method should be used with caution: incorrect datatype modifications may leave your network
* in an incorrect state. For example, {@code op(x(float),y(float)) -> op(x(double),y(float))} may not be
* supported by all ops.

View File

@ -382,7 +382,7 @@ public class SDBaseOps {
}
/**
* Cast the array to a new datatype - for example, Integer -> Float<br>
* Cast the array to a new datatype - for example, Integer -&gt; Float<br>
*
* @param arg Input variable to cast (NDARRAY type)
* @param datatype Datatype to cast to
@ -393,7 +393,7 @@ public class SDBaseOps {
}
/**
* Cast the array to a new datatype - for example, Integer -> Float<br>
* Cast the array to a new datatype - for example, Integer -&gt; Float<br>
*
* @param name name May be null. Name for the output variable
* @param arg Input variable to cast (NDARRAY type)
@ -654,7 +654,7 @@ public class SDBaseOps {
*
* @param x Input variable (NUMERIC type)
* @param partitions 1D input with values 0 to numPartitions-1 (INT type)
* @param numPartitions Number of partitions, >= 1
* @param numPartitions Number of partitions, &gt;= 1
*/
public SDVariable[] dynamicPartition(SDVariable x, SDVariable partitions, int numPartitions) {
SDValidation.validateNumerical("dynamicPartition", "x", x);
@ -676,7 +676,7 @@ public class SDBaseOps {
* @param names names May be null. Arrays of names for the output variables.
* @param x Input variable (NUMERIC type)
* @param partitions 1D input with values 0 to numPartitions-1 (INT type)
* @param numPartitions Number of partitions, >= 1
* @param numPartitions Number of partitions, &gt;= 1
*/
public SDVariable[] dynamicPartition(String[] names, SDVariable x, SDVariable partitions,
int numPartitions) {
@ -689,7 +689,7 @@ public class SDBaseOps {
/**
* Dynamically merge the specified input arrays into a single array, using the specified indices<br>
*
* @param indices Indices to use when merging. Must be >= 1, same length as input variables (INT type)
* @param indices Indices to use when merging. Must be &gt;= 1, same length as input variables (INT type)
* @param x Input variables. (NUMERIC type)
* @return output Merged output variable (NUMERIC type)
*/
@ -705,7 +705,7 @@ public class SDBaseOps {
* Dynamically merge the specified input arrays into a single array, using the specified indices<br>
*
* @param name name May be null. Name for the output variable
* @param indices Indices to use when merging. Must be >= 1, same length as input variables (INT type)
* @param indices Indices to use when merging. Must be &gt;= 1, same length as input variables (INT type)
* @param x Input variables. (NUMERIC type)
* @return output Merged output variable (NUMERIC type)
*/
@ -943,7 +943,7 @@ public class SDBaseOps {
}
/**
* Greater than operation: elementwise x > y<br>
* Greater than operation: elementwise x &gt; y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -957,7 +957,7 @@ public class SDBaseOps {
}
/**
* Greater than operation: elementwise x > y<br>
* Greater than operation: elementwise x &gt; y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -973,7 +973,7 @@ public class SDBaseOps {
}
/**
* Greater than operation: elementwise x > y<br>
* Greater than operation: elementwise x &gt; y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -993,7 +993,7 @@ public class SDBaseOps {
}
/**
* Greater than operation: elementwise x > y<br>
* Greater than operation: elementwise x &gt; y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -1015,7 +1015,7 @@ public class SDBaseOps {
}
/**
* Greater than or equals operation: elementwise x >= y<br>
* Greater than or equals operation: elementwise x &gt;= y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -1029,7 +1029,7 @@ public class SDBaseOps {
}
/**
* Greater than or equals operation: elementwise x >= y<br>
* Greater than or equals operation: elementwise x &gt;= y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -1045,7 +1045,7 @@ public class SDBaseOps {
}
/**
* Greater than or equal to operation: elementwise x >= y<br>
* Greater than or equal to operation: elementwise x &gt;= y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -1065,7 +1065,7 @@ public class SDBaseOps {
}
/**
* Greater than or equal to operation: elementwise x >= y<br>
* Greater than or equal to operation: elementwise x &gt;= y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -1232,7 +1232,7 @@ public class SDBaseOps {
}
/**
* Less than operation: elementwise x < y<br>
* Less than operation: elementwise x &lt; y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -1246,7 +1246,7 @@ public class SDBaseOps {
}
/**
* Less than operation: elementwise x < y<br>
* Less than operation: elementwise x &lt; y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -1262,7 +1262,7 @@ public class SDBaseOps {
}
/**
* Less than operation: elementwise x < y<br>
* Less than operation: elementwise x &lt; y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -1282,7 +1282,7 @@ public class SDBaseOps {
}
/**
* Less than operation: elementwise x < y<br>
* Less than operation: elementwise x &lt; y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -1304,7 +1304,7 @@ public class SDBaseOps {
}
/**
* Less than or equals operation: elementwise x <= y<br>
* Less than or equals operation: elementwise x &lt;= y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -1318,7 +1318,7 @@ public class SDBaseOps {
}
/**
* Less than or equals operation: elementwise x <= y<br>
* Less than or equals operation: elementwise x &lt;= y<br>
*
* Return boolean array with values true where satisfied, or false otherwise.<br>
*
@ -1334,7 +1334,7 @@ public class SDBaseOps {
}
/**
* Less than or equal to operation: elementwise x <= y<br>
* Less than or equal to operation: elementwise x &lt;= y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -1354,7 +1354,7 @@ public class SDBaseOps {
}
/**
* Less than or equal to operation: elementwise x <= y<br>
* Less than or equal to operation: elementwise x &lt;= y<br>
* If x and y arrays have equal shape, the output shape is the same as these inputs.<br>
*
* Note: supports broadcasting if x and y have different shapes and are broadcastable.<br>
@ -3590,7 +3590,7 @@ public class SDBaseOps {
/**
* Generate a sequence mask (with values 0 or 1) based on the specified lengths <br>
* Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)<br>
* {@code Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)}<br>
*
* @param lengths Lengths of the sequences (NUMERIC type)
* @param maxLen Maximum sequence length
@ -3604,7 +3604,7 @@ public class SDBaseOps {
/**
* Generate a sequence mask (with values 0 or 1) based on the specified lengths <br>
* Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)<br>
* {@code Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)}<br>
*
* @param name name May be null. Name for the output variable
* @param lengths Lengths of the sequences (NUMERIC type)
@ -3620,7 +3620,7 @@ public class SDBaseOps {
/**
* Generate a sequence mask (with values 0 or 1) based on the specified lengths <br>
* Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)<br>
* {@code Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)}<br>
*
* @param lengths Lengths of the sequences (NUMERIC type)
* @param maxLen Maximum sequence length (INT type)
@ -3635,7 +3635,7 @@ public class SDBaseOps {
/**
* Generate a sequence mask (with values 0 or 1) based on the specified lengths <br>
* Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)<br>
* {@code Specifically, out[i, ..., k, j] = (j < lengths[i, ..., k] ? 1.0 : 0.0)}<br>
*
* @param name name May be null. Name for the output variable
* @param lengths Lengths of the sequences (NUMERIC type)
@ -3761,7 +3761,7 @@ public class SDBaseOps {
* then slice(input, begin=[0,1], size=[2,1] will return:<br>
* [b]<br>
* [e]<br>
* Note that for each dimension i, begin[i] + size[i] <= input.size(i)<br>
* Note that for each dimension i, begin[i] + size[i] &lt;= input.size(i)<br>
*
* @param input input Variable to get subset of (NUMERIC type)
* @param begin Beginning index. Must be same length as rank of input array (Size: AtLeast(min=1))
@ -3783,7 +3783,7 @@ public class SDBaseOps {
* then slice(input, begin=[0,1], size=[2,1] will return:<br>
* [b]<br>
* [e]<br>
* Note that for each dimension i, begin[i] + size[i] <= input.size(i)<br>
* Note that for each dimension i, begin[i] + size[i] &lt;= input.size(i)<br>
*
* @param name name May be null. Name for the output variable
* @param input input Variable to get subset of (NUMERIC type)
@ -3807,7 +3807,7 @@ public class SDBaseOps {
* then slice(input, begin=[0,1], size=[2,1] will return:<br>
* [b]<br>
* [e]<br>
* Note that for each dimension i, begin[i] + size[i] <= input.size(i)<br>
* Note that for each dimension i, begin[i] + size[i] &lt;= input.size(i)<br>
*
* @param input input Variable to get subset of (NUMERIC type)
* @param begin Beginning index. Must be same length as rank of input array (INT type)
@ -3829,7 +3829,7 @@ public class SDBaseOps {
* then slice(input, begin=[0,1], size=[2,1] will return:<br>
* [b]<br>
* [e]<br>
* Note that for each dimension i, begin[i] + size[i] <= input.size(i)<br>
* Note that for each dimension i, begin[i] + size[i] &lt;= input.size(i)<br>
*
* @param name name May be null. Name for the output variable
* @param input input Variable to get subset of (NUMERIC type)

View File

@ -176,7 +176,7 @@ public class Evaluation extends BaseEvaluation<Evaluation> {
* Constructor to use for top N accuracy
*
* @param labels Labels for the classes (may be null)
* @param topN Value to use for top N accuracy calculation (<=1: standard accuracy). Note that with top N
* @param topN Value to use for top N accuracy calculation (&lt;=1: standard accuracy). Note that with top N
* accuracy, an example is considered 'correct' if the probability for the true class is one of the
* highest N values
*/
@ -1173,7 +1173,7 @@ public class Evaluation extends BaseEvaluation<Evaluation> {
/**
* False Alarm Rate (FAR) reflects rate of misclassified to classified records
* <a href="http://ro.ecu.edu.au/cgi/viewcontent.cgi?article=1058&context=isw">http://ro.ecu.edu.au/cgi/viewcontent.cgi?article=1058&context=isw</a><br>
* {@link }http://ro.ecu.edu.au/cgi/viewcontent.cgi?article=1058&context=isw}<br>
* Note: value returned will differ depending on number of classes and settings.<br>
* 1. For binary classification, if the positive class is set (via default value of 1, via constructor,
* or via {@link #setBinaryPositiveClass(Integer)}), the returned value will be for the specified positive class

View File

@ -34,7 +34,7 @@ public class BlasException extends Error {
}
/**
* Principal constructor - error message & error code
* Principal constructor - error message &amp; error code
* @param message the error message to put into the Exception
* @param errorCode the library error number
*/

View File

@ -28,15 +28,15 @@ public interface Lapack {
* LU decomposiiton of a matrix
* Factorize a matrix A
*
* The matrix A is overridden by the L & U combined.
* The matrix A is overridden by the L &amp; U combined.
* The permutation results are returned directly as a vector. To
* create the permutation matrix use getPFactor method
* To split out the L & U matrix use getLFactor and getUFactor methods
* To split out the L &amp; U matrix use getLFactor and getUFactor methods
*
* getrf = triangular factorization (TRF) of a general matrix (GE)
*
* @param A the input matrix, it will be overwritten with the factors
* @returns Permutation array
* @return Permutation array
* @throws Error - with a message to indicate failure (usu. bad params)
*/
INDArray getrf(INDArray A);
@ -53,7 +53,7 @@ public interface Lapack {
* matrix Q and an upper triangular R matrix
*
* @param A the input matrix, it will be overwritten with the factors
* @param The R array if null R is not returned
* @param R The R array if null R is not returned
* @throws Error - with a message to indicate failure (usu. bad params)
*/
void geqrf(INDArray A, INDArray R);
@ -71,8 +71,7 @@ public interface Lapack {
* lower L ( or upper U ) triangular matrix
*
* @param A the input matrix, it will be overwritten with the factors
* @param whether to return the upper (false) or lower factor
* @returns Permutation array
* @param lower whether to return the upper (false) or lower factor
* @throws Error - with a message to indicate failure (usu. bad params)
*/
void potrf(INDArray A, boolean lower);
@ -122,7 +121,7 @@ public interface Lapack {
*
* @param M - the size of the permutation matrix ( usu. the # rows in factored matrix )
* @param ipiv - the vector returned from a refactoring
* @returned the square permutation matrix - size is the M x M
* @return the square permutation matrix - size is the M x M
*/
INDArray getPFactor(int M, INDArray ipiv);
@ -131,8 +130,8 @@ public interface Lapack {
* extracts the L (lower triangular) matrix from the LU factor result
* L will be the same dimensions as A
*
* @param A - the combined L & U matrices returned from factorization
* @returned the lower triangular with unit diagonal
* @param A - the combined L &amp; U matrices returned from factorization
* @return the lower triangular with unit diagonal
*/
INDArray getLFactor(INDArray A);
@ -141,8 +140,8 @@ public interface Lapack {
* extracts the U (upper triangular) matrix from the LU factor result
* U will be n x n matrix where n = num cols in A
*
* @param A - the combined L & U matrices returned from factorization
* @returned the upper triangular matrix
* @param A - the combined L &amp; U matrices returned from factorization
* @return the upper triangular matrix
*/
INDArray getUFactor(INDArray A);

View File

@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray;
public interface Level1 {
/**
* computes a vector-vector dot product.
* @param n
* @param N
* @param alpha
* @param X
* @param Y
@ -65,7 +65,7 @@ public interface Level1 {
/**
* finds the element of a
* vector that has the largest absolute value.
* @param n the length to iterate for
* @param N the length to iterate for
* @param arr the array to get the max
* index for
* @param stride the stride for the array
@ -105,7 +105,7 @@ public interface Level1 {
/**
* computes a vector-scalar product and adds the result to a vector.
* @param n
* @param N
* @param alpha
* @param x
* @param y
@ -115,7 +115,7 @@ public interface Level1 {
/**
* computes a vector-scalar product and adds the result to a vector.
* y = a*x + y
* @param n number of operations
* @param N number of operations
* @param alpha
* @param x X
* @param offsetX offset of first element of X in buffer

View File

@ -1297,7 +1297,7 @@ public abstract class BaseDataBuffer implements DataBuffer {
if (offset() == 0) {
return wrappedBuffer().asIntBuffer();
} else
return wrappedBuffer().asIntBuffer().position((int) offset());
return (IntBuffer) wrappedBuffer().asIntBuffer().position((int) offset());
}
@Override
@ -1308,7 +1308,7 @@ public abstract class BaseDataBuffer implements DataBuffer {
if (offset() == 0) {
return wrappedBuffer().asLongBuffer();
} else
return wrappedBuffer().asLongBuffer().position((int) offset());
return (LongBuffer) wrappedBuffer().asLongBuffer().position((int) offset());
}
@Override
@ -1319,7 +1319,7 @@ public abstract class BaseDataBuffer implements DataBuffer {
if (offset() == 0) {
return wrappedBuffer().asDoubleBuffer();
} else {
return wrappedBuffer().asDoubleBuffer().position((int) (offset()));
return (DoubleBuffer) wrappedBuffer().asDoubleBuffer().position((int) (offset()));
}
}
@ -1331,7 +1331,8 @@ public abstract class BaseDataBuffer implements DataBuffer {
if (offset() == 0) {
return wrappedBuffer().asFloatBuffer();
} else {
return wrappedBuffer().asFloatBuffer().position((int) (offset()));
return (FloatBuffer) wrappedBuffer().asFloatBuffer()
.position((int) (offset()));
}
}

View File

@ -34,7 +34,7 @@ import java.util.*;
public class NDArrayCreationUtil {
private NDArrayCreationUtil() {}
/** Get an array of INDArrays (2d) all with the specified shape. Pair<INDArray,String> returned to aid
/** Get an array of INDArrays (2d) all with the specified shape. {@code Pair<INDArray,String>} returned to aid
* debugging: String contains information on how to reproduce the matrix (i.e., which function, and arguments)
* Each NDArray in the returned array has been obtained by applying an operation such as transpose, tensorAlongDimension,
* etc to an original array.

View File

@ -302,7 +302,7 @@ public interface DataSet extends Iterable<org.nd4j.linalg.dataset.DataSet>, Seri
* Get the example metadata, or null if no metadata has been set
*
* @return List of metadata instances
* @see {@link #getExampleMetaData(Class)} for convenience method for types
* {@link #getExampleMetaData(Class)} for convenience method for types
*/
List<Serializable> getExampleMetaData();

View File

@ -180,7 +180,7 @@ public interface MultiDataSet extends Serializable {
* Get the example metadata, or null if no metadata has been set
*
* @return List of metadata instances
* @see {@link #getExampleMetaData(Class)} for convenience method for types
* {@link #getExampleMetaData(Class)} for convenience method for types
*/
List<Serializable> getExampleMetaData();

View File

@ -35,23 +35,27 @@ import java.util.*;
public class ND4JTestUtils {
private ND4JTestUtils(){ }
private ND4JTestUtils() {
}
@AllArgsConstructor
@Data
public static class ComparisonResult {
List<Triple<File,File,Boolean>> allResults;
List<Triple<File,File,Boolean>> passed;
List<Triple<File,File,Boolean>> failed;
List<Triple<File, File, Boolean>> allResults;
List<Triple<File, File, Boolean>> passed;
List<Triple<File, File, Boolean>> failed;
List<File> skippedDir1;
List<File> skippedDir2;
}
/**
* A function for use with {@link #validateSerializedArrays(File, File, boolean, BiFunction)} using {@code INDArray#equals(Object)}
* A function for use with {@link #validateSerializedArrays(File, File, boolean, BiFunction)}
* using {@code INDArray#equals(Object)}
*/
public static class EqualsFn implements BiFunction<INDArray,INDArray,Boolean> {
public static class EqualsFn implements BiFunction<INDArray, INDArray, Boolean> {
@Override
public Boolean apply(INDArray i1, INDArray i2) {
return i1.equals(i2);
@ -59,10 +63,12 @@ public class ND4JTestUtils {
}
/**
* A function for use with {@link #validateSerializedArrays(File, File, boolean, BiFunction)} using {@link INDArray#equalsWithEps(Object, double)}
* A function for use with {@link #validateSerializedArrays(File, File, boolean, BiFunction)}
* using {@link INDArray#equalsWithEps(Object, double)}
*/
@AllArgsConstructor
public static class EqualsWithEpsFn implements BiFunction<INDArray,INDArray,Boolean> {
public static class EqualsWithEpsFn implements BiFunction<INDArray, INDArray, Boolean> {
private final double eps;
@Override
@ -72,90 +78,99 @@ public class ND4JTestUtils {
}
/**
* Scan the specified directories for matching files (i.e., same path relative to their respective root directories)
* and compare the contents using INDArray.equals (via {@link EqualsFn}
* Assumes the saved files represent INDArrays saved with {@link Nd4j#saveBinary(INDArray, File)}
* Scan the specified directories for matching files (i.e., same path relative to their respective
* root directories) and compare the contents using INDArray.equals (via {@link EqualsFn} Assumes
* the saved files represent INDArrays saved with {@link Nd4j#saveBinary(INDArray, File)}
*
* @param dir1 First directory
* @param dir2 Second directory
* @param recursive Whether to search recursively (i.e., include files in subdirectories
* @return Comparison results
*/
public static ComparisonResult validateSerializedArrays(File dir1, File dir2, boolean recursive) throws Exception {
public static ComparisonResult validateSerializedArrays(File dir1, File dir2, boolean recursive)
throws Exception {
return validateSerializedArrays(dir1, dir2, recursive, new EqualsFn());
}
/**
* Scan the specified directories for matching files (i.e., same path relative to their respective root directories)
* and compare the contents using a provided function.<br>
* Assumes the saved files represent INDArrays saved with {@link Nd4j#saveBinary(INDArray, File)}
* Scan the specified directories for matching files (i.e., same path relative to their respective
* root directories) and compare the contents using a provided function.<br> Assumes the saved
* files represent INDArrays saved with {@link Nd4j#saveBinary(INDArray, File)}
*
* @param dir1 First directory
* @param dir2 Second directory
* @param recursive Whether to search recursively (i.e., include files in subdirectories
* @return Comparison results
*/
public static ComparisonResult validateSerializedArrays(File dir1, File dir2, boolean recursive, BiFunction<INDArray,INDArray,Boolean> evalFn) throws Exception {
public static ComparisonResult validateSerializedArrays(File dir1, File dir2, boolean recursive,
BiFunction<INDArray, INDArray, Boolean> evalFn) throws Exception {
File[] f1 = FileUtils.listFiles(dir1, null, recursive).toArray(new File[0]);
File[] f2 = FileUtils.listFiles(dir2, null, recursive).toArray(new File[0]);
Preconditions.checkState(f1.length > 0, "No files found for directory 1: %s", dir1.getAbsolutePath() );
Preconditions.checkState(f2.length > 0, "No files found for directory 2: %s", dir2.getAbsolutePath() );
Preconditions.checkState(f1.length > 0, "No files found for directory 1: %s",
dir1.getAbsolutePath());
Preconditions.checkState(f2.length > 0, "No files found for directory 2: %s",
dir2.getAbsolutePath());
Map<String,File> relativized1 = new HashMap<>();
Map<String,File> relativized2 = new HashMap<>();
Map<String, File> relativized1 = new HashMap<>();
Map<String, File> relativized2 = new HashMap<>();
URI u = dir1.toURI();
for(File f : f1){
if(!f.isFile())
for (File f : f1) {
if (!f.isFile()) {
continue;
}
String relative = u.relativize(f.toURI()).getPath();
relativized1.put(relative, f);
}
u = dir2.toURI();
for(File f : f2){
if(!f.isFile())
for (File f : f2) {
if (!f.isFile()) {
continue;
}
String relative = u.relativize(f.toURI()).getPath();
relativized2.put(relative, f);
}
List<File> skipped1 = new ArrayList<>();
for(String s : relativized1.keySet()){
if(!relativized2.containsKey(s)){
for (String s : relativized1.keySet()) {
if (!relativized2.containsKey(s)) {
skipped1.add(relativized1.get(s));
}
}
List<File> skipped2 = new ArrayList<>();
for(String s : relativized2.keySet()){
if(!relativized1.containsKey(s)){
for (String s : relativized2.keySet()) {
if (!relativized1.containsKey(s)) {
skipped2.add(relativized1.get(s));
}
}
List<Triple<File,File,Boolean>> allResults = new ArrayList<>();
List<Triple<File,File,Boolean>> passed = new ArrayList<>();
List<Triple<File,File,Boolean>> failed = new ArrayList<>();
for(Map.Entry<String,File> e : relativized1.entrySet()){
List<Triple<File, File, Boolean>> allResults = new ArrayList<>();
List<Triple<File, File, Boolean>> passed = new ArrayList<>();
List<Triple<File, File, Boolean>> failed = new ArrayList<>();
for (Map.Entry<String, File> e : relativized1.entrySet()) {
File file1 = e.getValue();
File file2 = relativized2.get(e.getKey());
if(file2 == null)
if (file2 == null) {
continue;
}
INDArray i1 = Nd4j.readBinary(file1);
INDArray i2 = Nd4j.readBinary(file2);
boolean b = evalFn.apply(i1, i2);
Triple<File,File,Boolean> t = new Triple<>(file1, file2, b);
Triple<File, File, Boolean> t = new Triple<>(file1, file2, b);
allResults.add(t);
if(b){
if (b) {
passed.add(t);
} else {
failed.add(t);
}
}
Comparator<Triple<File,File,Boolean>> c = new Comparator<Triple<File, File, Boolean>>() {
Comparator<Triple<File, File, Boolean>> c = new Comparator<Triple<File, File, Boolean>>() {
@Override
public int compare(Triple<File, File, Boolean> o1, Triple<File, File, Boolean> o2) {
return o1.getFirst().compareTo(o2.getFirst());
@ -168,7 +183,6 @@ public class ND4JTestUtils {
Collections.sort(skipped1);
Collections.sort(skipped2);
return new ComparisonResult(allResults, passed, failed, skipped1, skipped2);
}
}

View File

@ -211,7 +211,7 @@ public class EmnistDataSetIterator extends BaseDatasetIterator {
}
/**
* Get the labels as a List<String>
* Get the labels as a {@code List<String>}
*
* @return Labels
*/
@ -244,7 +244,7 @@ public class EmnistDataSetIterator extends BaseDatasetIterator {
}
/**
* Get the label assignments for the given set as a List<String>
* Get the label assignments for the given set as a {@code List<String>}
*
* @param dataSet DataSet to get the label assignment for
* @return Label assignment and given dataset

View File

@ -31,59 +31,91 @@ import java.util.Random;
public class LFWDataSetIterator extends RecordReaderDataSetIterator {
/** Loads subset of images with given imgDim returned by the generator. */
/**
* Loads subset of images with given imgDim returned by the generator.
*/
public LFWDataSetIterator(int[] imgDim) {
this(LFWLoader.SUB_NUM_IMAGES, LFWLoader.SUB_NUM_IMAGES, imgDim, LFWLoader.SUB_NUM_LABELS, false,
this(LFWLoader.SUB_NUM_IMAGES, LFWLoader.SUB_NUM_IMAGES, imgDim, LFWLoader.SUB_NUM_LABELS,
false,
new ParentPathLabelGenerator(), true, 1, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples returned by the generator. */
/**
* Loads images with given batchSize, numExamples returned by the generator.
*/
public LFWDataSetIterator(int batchSize, int numExamples) {
this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS},
this(batchSize, numExamples, new int[]{LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS},
LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, imgDim returned by the generator. */
/**
* Loads images with given batchSize, numExamples, imgDim returned by the generator.
*/
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim) {
this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true, 1, null,
this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, true,
1, null,
new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, imgDim, useSubset, returned by the generator. */
/**
* Loads images with given batchSize, imgDim, useSubset, returned by the generator.
*/
public LFWDataSetIterator(int batchSize, int[] imgDim, boolean useSubset) {
this(batchSize, useSubset ? LFWLoader.SUB_NUM_IMAGES : LFWLoader.NUM_IMAGES, imgDim,
useSubset ? LFWLoader.SUB_NUM_LABELS : LFWLoader.NUM_LABELS, useSubset, LFWLoader.LABEL_PATTERN,
useSubset ? LFWLoader.SUB_NUM_LABELS : LFWLoader.NUM_LABELS, useSubset,
LFWLoader.LABEL_PATTERN,
true, 1, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, imgDim, train, & splitTrainTest returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, boolean train, double splitTrainTest) {
this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN, train,
/**
* Loads images with given batchSize, numExamples, imgDim, train, &amp; splitTrainTest returned
* by the generator.
*/
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, boolean train,
double splitTrainTest) {
this(batchSize, numExamples, imgDim, LFWLoader.NUM_LABELS, false, LFWLoader.LABEL_PATTERN,
train,
splitTrainTest, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, numLabels, train, & splitTrainTest returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int numLabels, boolean train, double splitTrainTest) {
this(batchSize, numExamples, new int[] {LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS}, numLabels,
/**
* Loads images with given batchSize, numExamples, numLabels, train, &amp; splitTrainTest
* returned by the generator.
*/
public LFWDataSetIterator(int batchSize, int numExamples, int numLabels, boolean train,
double splitTrainTest) {
this(batchSize, numExamples, new int[]{LFWLoader.HEIGHT, LFWLoader.WIDTH, LFWLoader.CHANNELS},
numLabels,
false, null, train, splitTrainTest, null, new Random(System.currentTimeMillis()));
}
/** Loads images with given batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
/**
* Loads images with given batchSize, numExamples, imgDim, numLabels, useSubset, train,
* splitTrainTest &amp; Random returned by the generator.
*/
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels,
boolean useSubset,
boolean train, double splitTrainTest, Random rng) {
this(batchSize, numExamples, imgDim, numLabels, useSubset, LFWLoader.LABEL_PATTERN, train, splitTrainTest, null,
this(batchSize, numExamples, imgDim, numLabels, useSubset, LFWLoader.LABEL_PATTERN, train,
splitTrainTest, null,
rng);
}
/** Loads images with given batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
/**
* Loads images with given batchSize, numExamples, imgDim, numLabels, useSubset, train,
* splitTrainTest &amp; Random returned by the generator.
*/
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels,
boolean useSubset,
PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train, splitTrainTest, null, rng);
this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train,
splitTrainTest, null, rng);
}
/**
* Create LFW data specific iterator
*
* @param batchSize the batch size of the examples
* @param numExamples the overall number of examples
* @param imgDim an array of height, width and channels
@ -93,13 +125,14 @@ public class LFWDataSetIterator extends RecordReaderDataSetIterator {
* @param train true if use train value
* @param splitTrainTest the percentage to split data for train and remainder goes to test
* @param imageTransform how to transform the image
* @param rng random number to lock in batch shuffling
* */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
*/
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels,
boolean useSubset,
PathLabelGenerator labelGenerator, boolean train, double splitTrainTest,
ImageTransform imageTransform, Random rng) {
super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples, imgDim,
super(new LFWLoader(imgDim, imageTransform, useSubset).getRecordReader(batchSize, numExamples,
imgDim,
numLabels, labelGenerator, train, splitTrainTest, rng), batchSize, 1, numLabels);
}

View File

@ -12,12 +12,15 @@ configurations.archives.artifacts.with { archives ->
dependencies {
//Todo clean this
api platform(project(":cavis-common-platform"))
api "org.bytedeco:javacpp:1.5.7" //for some reason we needed to apply version numbers here, they do not end up in POM otherwise
//api "org.bytedeco:javacpp:1.5.7" //for some reason we needed to apply version numbers here, they do not end up in POM otherwise
api "com.fasterxml.jackson.datatype:jackson-datatype-joda:2.10.5"
api 'org.slf4j:slf4j-simple:2.0.3'
api 'org.slf4j:slf4j-api:2.0.3'
//api group: "org.bytedeco", name: "javacpp", classifier: "linux-x64_86"
//TODO for the two below.. either platform specific uber jars or a single big one with all platforms
api group: "org.bytedeco", name: "javacpp", version: "1.5.7", classifier: "linux-x86_64"
//api group: "org.bytedeco", name: "javacpp", version: "1.5.7"
// api group: 'net.brutex.cavis-native', name: 'cavis-native-lib', version: '1.0.0-SNAPSHOT', classifier: "linux-x86_64-avx2-cpu"
//api group: 'net.brutex.cavis-native', name: 'cavis-native-lib', version: '1.0.0-SNAPSHOT'
rootProject.getAllprojects().each { Project sproj ->
if(!sproj.name.equals(name) && !sproj.name.equals("cavis-common-platform")
&& !sproj.name.equals("Cavis")

View File

@ -256,7 +256,7 @@ public class NDArrayMessage implements Serializable {
String messageId = UUID.randomUUID().toString();
for (int i = 0; i < ret.length; i++) {
//data: only grab a chunk of the data
ByteBuffer view = wholeBuffer.byteBuffer().asReadOnlyBuffer().position(i * chunkSize);
ByteBuffer view = (ByteBuffer) wholeBuffer.byteBuffer().asReadOnlyBuffer().position(i * chunkSize);
view.limit(Math.min(i * chunkSize + chunkSize, wholeBuffer.capacity()));
view.order(ByteOrder.nativeOrder());
view = view.slice();

View File

@ -664,8 +664,8 @@ public class RoutedTransport extends BaseTransport {
public static class RemoteConnectionBuilder {
private final Object locker = new Object();
private final AtomicBoolean activated = new AtomicBoolean();
private Object locker = new Object();
private AtomicBoolean activated = new AtomicBoolean();
}
}