2019-06-06 15:21:15 +03:00
|
|
|
/*******************************************************************************
|
|
|
|
* Copyright (c) 2015-2018 Skymind, Inc.
|
|
|
|
*
|
|
|
|
* This program and the accompanying materials are made available under the
|
|
|
|
* terms of the Apache License, Version 2.0 which is available at
|
|
|
|
* https://www.apache.org/licenses/LICENSE-2.0.
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
|
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
|
|
* License for the specific language governing permissions and limitations
|
|
|
|
* under the License.
|
|
|
|
*
|
|
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
|
|
******************************************************************************/
|
|
|
|
|
|
|
|
package org.datavec.api.writable;
|
|
|
|
|
|
|
|
|
Version upgrades (#199)
* DataVec fixes for Jackson version upgrade
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J jackson updates + databind version 2.9.9.3
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Shade snakeyaml along with jackson
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Version fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Switch DataVec legacy JSON format handling to mixins
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Next set of fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Cleanup for legacy JSON mapping
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Upgrade commons compress to 1.18; small test fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* New Jackson backward compatibility for DL4J - Round 1
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* New Jackson backward compatibility for DL4J - Round 2
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* More fixes, all but legacy custom passing
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Provide an upgrade path for custom layers for models in pre-1.0.0-beta JSON format
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Legacy deserialization cleanup
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Small amount of polish - legacy JSON
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Upgrade guava version
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* IEvaluation legacy format deserialization fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Upgrade play version to 2.7.3
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Update nd4j-parameter-server-status to new Play API
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Update DL4J UI for new play version
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* More play framework updates
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Small fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Remove Spark 1/2 adapter code from DataVec
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* datavec-spark dependency cleanup
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 1
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 2
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 3
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 4
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Test fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Another fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Breeze upgrade, dependency cleanup
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Add Scala 2.12 version to pom.xml
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* change-scala-versions.sh - add scala 2.12, remove 2.10
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Move Spark version properties to parent pom (now that only one spark version is supported)
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DataVec Play fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* datavec play dependency fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Clean up old spark/jackson stuff
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Cleanup jackson unused dependencies
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Dropping redundant dependency
Signed-off-by: Alexander Stoyakin <alexander.stoyakin@gmail.com>
* Removed scalaxy dependency
Signed-off-by: Alexander Stoyakin <alexander.stoyakin@gmail.com>
* DataVec fixes for Jackson version upgrade
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J jackson updates + databind version 2.9.9.3
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Shade snakeyaml along with jackson
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Version fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Switch DataVec legacy JSON format handling to mixins
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Next set of fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Cleanup for legacy JSON mapping
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Upgrade commons compress to 1.18; small test fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* New Jackson backward compatibility for DL4J - Round 1
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* New Jackson backward compatibility for DL4J - Round 2
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* More fixes, all but legacy custom passing
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Provide an upgrade path for custom layers for models in pre-1.0.0-beta JSON format
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Legacy deserialization cleanup
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Small amount of polish - legacy JSON
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Upgrade guava version
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* IEvaluation legacy format deserialization fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Upgrade play version to 2.7.3
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Update nd4j-parameter-server-status to new Play API
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Update DL4J UI for new play version
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* More play framework updates
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Small fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Remove Spark 1/2 adapter code from DataVec
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* datavec-spark dependency cleanup
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 1
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 2
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 3
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DL4J spark updates, pt 4
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Test fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Another fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Breeze upgrade, dependency cleanup
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Add Scala 2.12 version to pom.xml
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* change-scala-versions.sh - add scala 2.12, remove 2.10
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Move Spark version properties to parent pom (now that only one spark version is supported)
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DataVec Play fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* datavec play dependency fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Clean up old spark/jackson stuff
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Cleanup jackson unused dependencies
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Add shaded guava
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Dropping redundant dependency
Signed-off-by: Alexander Stoyakin <alexander.stoyakin@gmail.com>
* Removed scalaxy dependency
Signed-off-by: Alexander Stoyakin <alexander.stoyakin@gmail.com>
* Ensure not possible to import pre-shaded classes, and remove direct guava dependencies in favor of shaded
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* ND4J Shaded guava import fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* DataVec and DL4J guava shading
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Arbiter, RL4J fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Build fixed
Signed-off-by: Alexander Stoyakin <alexander.stoyakin@gmail.com>
* Fix dependency
Signed-off-by: Alexander Stoyakin <alexander.stoyakin@gmail.com>
* Fix bad merge
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Jackson shading fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Set play secret, datavec-spark-inference-server
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Fix for datavec-spark-inference-server
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Arbiter fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Arbiter fixes
Signed-off-by: AlexDBlack <blacka101@gmail.com>
* Small test fix
Signed-off-by: AlexDBlack <blacka101@gmail.com>
2019-08-30 14:35:27 +10:00
|
|
|
import org.nd4j.shade.guava.math.DoubleMath;
|
2019-06-06 15:21:15 +03:00
|
|
|
import org.datavec.api.io.WritableComparable;
|
|
|
|
import org.datavec.api.io.WritableComparator;
|
|
|
|
import org.nd4j.shade.jackson.annotation.JsonProperty;
|
|
|
|
|
|
|
|
import java.io.DataInput;
|
|
|
|
import java.io.DataOutput;
|
|
|
|
import java.io.IOException;
|
|
|
|
|
|
|
|
/** A WritableComparable for ints. */
|
|
|
|
public class IntWritable implements WritableComparable {
|
|
|
|
|
|
|
|
private int value;
|
|
|
|
|
|
|
|
public IntWritable() {}
|
|
|
|
|
|
|
|
public IntWritable(@JsonProperty("value") int value) {
|
|
|
|
set(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Set the value of this IntWritable. */
|
|
|
|
public void set(int value) {
|
|
|
|
this.value = value;
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Return the value of this IntWritable. */
|
|
|
|
public int get() {
|
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
public void readFields(DataInput in) throws IOException {
|
|
|
|
value = in.readInt();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public void writeType(DataOutput out) throws IOException {
|
|
|
|
out.writeShort(WritableType.Int.typeIdx());
|
|
|
|
}
|
|
|
|
|
|
|
|
public void write(DataOutput out) throws IOException {
|
|
|
|
out.writeInt(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
public boolean fuzzyEquals(Writable o, double tolerance) {
|
|
|
|
double other;
|
|
|
|
if (o instanceof IntWritable){
|
|
|
|
other = ((IntWritable) o).toDouble();
|
|
|
|
} else if (o instanceof LongWritable) {
|
|
|
|
other = ((LongWritable) o).toDouble();
|
|
|
|
} else if (o instanceof ByteWritable) {
|
|
|
|
other = ((ByteWritable) o).toDouble();
|
|
|
|
} else if (o instanceof DoubleWritable) {
|
|
|
|
other = ((DoubleWritable) o).toDouble();
|
|
|
|
} else if (o instanceof FloatWritable) {
|
|
|
|
other = ((FloatWritable) o).toDouble();
|
|
|
|
} else { return false; }
|
|
|
|
return DoubleMath.fuzzyEquals(this.value, other, tolerance);
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Returns true iff <code>o</code> is a IntWritable with the same value. */
|
|
|
|
public boolean equals(Object o) {
|
|
|
|
if (o instanceof ByteWritable){
|
|
|
|
ByteWritable other = (ByteWritable) o;
|
|
|
|
return this.value == other.get();
|
|
|
|
}
|
|
|
|
if (o instanceof IntWritable) {
|
|
|
|
IntWritable other = (IntWritable) o;
|
|
|
|
return this.value == other.get();
|
|
|
|
}
|
|
|
|
if (o instanceof LongWritable) {
|
|
|
|
LongWritable other = (LongWritable) o;
|
|
|
|
return this.value == other.get();
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public int hashCode() {
|
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Compares two IntWritables. */
|
|
|
|
public int compareTo(Object o) {
|
|
|
|
int thisValue = this.value;
|
|
|
|
int thatValue = ((IntWritable) o).value;
|
|
|
|
return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
|
|
|
|
}
|
|
|
|
|
|
|
|
public String toString() {
|
|
|
|
return Integer.toString(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
/** A Comparator optimized for IntWritable. */
|
|
|
|
public static class Comparator extends WritableComparator {
|
|
|
|
public Comparator() {
|
|
|
|
super(IntWritable.class);
|
|
|
|
}
|
|
|
|
|
|
|
|
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
|
|
|
|
int thisValue = readInt(b1, s1);
|
|
|
|
int thatValue = readInt(b2, s2);
|
|
|
|
return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static { // register this comparator
|
|
|
|
WritableComparator.define(IntWritable.class, new Comparator());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public double toDouble() {
|
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public float toFloat() {
|
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public int toInt() {
|
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public long toLong() {
|
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public WritableType getType() {
|
|
|
|
return WritableType.Int;
|
|
|
|
}
|
|
|
|
}
|