nd4j-jackson dependency fix (#398)
* Remove old nd4j-jackson dependencies Signed-off-by: Alex Black <blacka101@gmail.com> * Fix use of old/deprecated JSON serializer Signed-off-by: Alex Black <blacka101@gmail.com> * Fix deserialization Signed-off-by: Alex Black <blacka101@gmail.com> * Delete test using deleted ser/de classes Signed-off-by: Alex Black <blacka101@gmail.com> * Delete another copy of old test Signed-off-by: Alex Black <blacka101@gmail.com>master
Alex Black
GitHub
parent
191bda3228
commit
fe516ae6cf
|
|
@ -30,11 +30,6 @@
|
|||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.datavec</groupId>
|
||||
<artifactId>datavec-spark-inference-server_2.11</artifactId>
|
||||
|
|
|
|||
|
|
@ -41,11 +41,6 @@
|
|||
<artifactId>deeplearning4j-nearestneighbors-model</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -89,12 +89,6 @@
|
|||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- oshi: Used for collecting system information for memory crash dump reporting -->
|
||||
<dependency>
|
||||
<groupId>com.github.oshi</groupId>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
/* ******************************************************************************
|
||||
* Copyright (c) 2020 Konduit K.K.
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
package org.deeplearning4j.nn.conf.layers.objdetect;
|
||||
|
||||
import org.nd4j.linalg.api.buffer.DataBuffer;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer;
|
||||
import org.nd4j.shade.jackson.core.JsonParser;
|
||||
import org.nd4j.shade.jackson.core.JsonProcessingException;
|
||||
import org.nd4j.shade.jackson.databind.DeserializationContext;
|
||||
import org.nd4j.shade.jackson.databind.JsonDeserializer;
|
||||
import org.nd4j.shade.jackson.databind.JsonNode;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Custom deserializer to handle change in format between beta6 (and earlier) and later versions
|
||||
*
|
||||
* @author Alex Black
|
||||
*/
|
||||
public class BoundingBoxesDeserializer extends JsonDeserializer<INDArray> {
|
||||
@Override
|
||||
public INDArray deserialize(JsonParser jp, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
|
||||
JsonNode node = jp.getCodec().readTree(jp);
|
||||
if(node.has("dataBuffer")){
|
||||
//Must be legacy format serialization
|
||||
JsonNode arr = node.get("dataBuffer");
|
||||
int rank = node.get("rankField").asInt();
|
||||
int numElements = node.get("numElements").asInt();
|
||||
int offset = node.get("offsetField").asInt();
|
||||
JsonNode shape = node.get("shapeField");
|
||||
JsonNode stride = node.get("strideField");
|
||||
int[] shapeArr = new int[rank];
|
||||
int[] strideArr = new int[rank];
|
||||
DataBuffer buff = Nd4j.createBuffer(numElements);
|
||||
for (int i = 0; i < numElements; i++) {
|
||||
buff.put(i, arr.get(i).asDouble());
|
||||
}
|
||||
|
||||
String ordering = node.get("orderingField").asText();
|
||||
for (int i = 0; i < rank; i++) {
|
||||
shapeArr[i] = shape.get(i).asInt();
|
||||
strideArr[i] = stride.get(i).asInt();
|
||||
}
|
||||
|
||||
return Nd4j.create(buff, shapeArr, strideArr, offset, ordering.charAt(0));
|
||||
}
|
||||
//Standard/new format
|
||||
return new NDArrayTextDeSerializer().deserialize(node);
|
||||
}
|
||||
}
|
||||
|
|
@ -34,10 +34,9 @@ import org.nd4j.linalg.api.ndarray.INDArray;
|
|||
import org.nd4j.linalg.learning.regularization.Regularization;
|
||||
import org.nd4j.linalg.lossfunctions.ILossFunction;
|
||||
import org.nd4j.linalg.lossfunctions.impl.LossL2;
|
||||
import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize;
|
||||
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
|
||||
import org.nd4j.shade.serde.jackson.VectorDeSerializer;
|
||||
import org.nd4j.shade.serde.jackson.VectorSerializer;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
|
@ -77,8 +76,8 @@ public class Yolo2OutputLayer extends org.deeplearning4j.nn.conf.layers.Layer {
|
|||
private double lambdaNoObj;
|
||||
private ILossFunction lossPositionScale;
|
||||
private ILossFunction lossClassPredictions;
|
||||
@JsonSerialize(using = VectorSerializer.class)
|
||||
@JsonDeserialize(using = VectorDeSerializer.class)
|
||||
@JsonSerialize(using = NDArrayTextSerializer.class)
|
||||
@JsonDeserialize(using = BoundingBoxesDeserializer.class)
|
||||
private INDArray boundingBoxes;
|
||||
|
||||
private Yolo2OutputLayer() {
|
||||
|
|
|
|||
|
|
@ -44,11 +44,6 @@
|
|||
<artifactId>deeplearning4j-nlp</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${nd4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ public class NDArrayTextDeSerializer extends JsonDeserializer<INDArray> {
|
|||
@Override
|
||||
public INDArray deserialize(JsonParser jp, DeserializationContext deserializationContext) throws IOException {
|
||||
JsonNode n = jp.getCodec().readTree(jp);
|
||||
return deserialize(n);
|
||||
}
|
||||
|
||||
public INDArray deserialize(JsonNode n){
|
||||
|
||||
//First: check for backward compatilibity (RowVectorSerializer/Deserializer)
|
||||
if(!n.has("dataType")){
|
||||
|
|
|
|||
|
|
@ -274,12 +274,6 @@
|
|||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-api</artifactId>
|
||||
|
|
|
|||
|
|
@ -76,12 +76,6 @@
|
|||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Shaded version of Jackson -->
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
/*******************************************************************************
|
||||
* Copyright (c) 2015-2018 Skymind, Inc.
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
package org.nd4j.serde.jackson;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.nd4j.linalg.BaseNd4jTest;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.nd4j.linalg.factory.Nd4jBackend;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.module.SimpleModule;
|
||||
import org.nd4j.shade.serde.jackson.shaded.NDArrayDeSerializer;
|
||||
import org.nd4j.shade.serde.jackson.shaded.NDArraySerializer;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Created by agibsonccc on 6/23/16.
|
||||
*/
|
||||
public class NdArraySerializerTest extends BaseNd4jTest {
|
||||
private static ObjectMapper objectMapper;
|
||||
|
||||
public NdArraySerializerTest(Nd4jBackend backend) {
|
||||
super(backend);
|
||||
}
|
||||
|
||||
@Override
|
||||
public char ordering() {
|
||||
return 'c';
|
||||
}
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() {
|
||||
objectMapper = objectMapper();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSerde() throws Exception {
|
||||
String json = objectMapper.writeValueAsString(Nd4j.create(2, 2));
|
||||
INDArray assertion = Nd4j.create(2, 2);
|
||||
INDArray test = objectMapper.readValue(json, INDArray.class);
|
||||
assertEquals(assertion, test);
|
||||
}
|
||||
|
||||
private static ObjectMapper objectMapper() {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
SimpleModule nd4j = new SimpleModule("nd4j");
|
||||
nd4j.addDeserializer(INDArray.class, new NDArrayDeSerializer());
|
||||
nd4j.addSerializer(INDArray.class, new NDArraySerializer());
|
||||
mapper.registerModule(nd4j);
|
||||
return mapper;
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
/*******************************************************************************
|
||||
* Copyright (c) 2015-2018 Skymind, Inc.
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
package org.nd4j.serde.jackson;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.nd4j.linalg.BaseNd4jTest;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.nd4j.linalg.factory.Nd4jBackend;
|
||||
import org.nd4j.shade.jackson.databind.ObjectMapper;
|
||||
import org.nd4j.shade.jackson.databind.module.SimpleModule;
|
||||
import org.nd4j.shade.serde.jackson.VectorDeSerializer;
|
||||
import org.nd4j.shade.serde.jackson.VectorSerializer;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Created by agibsonccc on 6/23/16.
|
||||
*/
|
||||
public class VectorSerializeTest extends BaseNd4jTest {
|
||||
private static ObjectMapper objectMapper;
|
||||
|
||||
public VectorSerializeTest(Nd4jBackend backend) {
|
||||
super(backend);
|
||||
}
|
||||
|
||||
@Override
|
||||
public char ordering() {
|
||||
return 'c';
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() {
|
||||
objectMapper = objectMapper();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testSerde() throws Exception {
|
||||
String json = objectMapper.writeValueAsString(Nd4j.create(2, 2));
|
||||
INDArray assertion = Nd4j.create(2, 2);
|
||||
INDArray test = objectMapper.readValue(json, INDArray.class);
|
||||
assertEquals(assertion, test);
|
||||
}
|
||||
|
||||
|
||||
private static ObjectMapper objectMapper() {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
SimpleModule nd4j = new SimpleModule("nd4j");
|
||||
nd4j.addDeserializer(INDArray.class, new VectorDeSerializer());
|
||||
nd4j.addSerializer(INDArray.class, new VectorSerializer());
|
||||
mapper.registerModule(nd4j);
|
||||
return mapper;
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -180,11 +180,6 @@
|
|||
<artifactId>nd4j-aeron</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-jackson</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.nd4j</groupId>
|
||||
<artifactId>nd4j-kryo_2.11</artifactId>
|
||||
|
|
|
|||
Loading…
Reference in New Issue