nd4j-jackson dependency fix (#398)

* Remove old nd4j-jackson dependencies

Signed-off-by: Alex Black <blacka101@gmail.com>

* Fix use of old/deprecated JSON serializer

Signed-off-by: Alex Black <blacka101@gmail.com>

* Fix deserialization

Signed-off-by: Alex Black <blacka101@gmail.com>

* Delete test using deleted ser/de classes

Signed-off-by: Alex Black <blacka101@gmail.com>

* Delete another copy of old test

Signed-off-by: Alex Black <blacka101@gmail.com>
master
Alex Black 2020-04-20 22:33:11 +10:00 committed by GitHub
parent 191bda3228
commit fe516ae6cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 72 additions and 186 deletions

View File

@ -30,11 +30,6 @@
<dependencies>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>org.datavec</groupId>
<artifactId>datavec-spark-inference-server_2.11</artifactId>

View File

@ -41,11 +41,6 @@
<artifactId>deeplearning4j-nearestneighbors-model</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${nd4j.version}</version>
</dependency>
</dependencies>

View File

@ -89,12 +89,6 @@
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${nd4j.version}</version>
</dependency>
<!-- oshi: Used for collecting system information for memory crash dump reporting -->
<dependency>
<groupId>com.github.oshi</groupId>

View File

@ -0,0 +1,65 @@
/* ******************************************************************************
* Copyright (c) 2020 Konduit K.K.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.deeplearning4j.nn.conf.layers.objdetect;
import org.nd4j.linalg.api.buffer.DataBuffer;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer;
import org.nd4j.shade.jackson.core.JsonParser;
import org.nd4j.shade.jackson.core.JsonProcessingException;
import org.nd4j.shade.jackson.databind.DeserializationContext;
import org.nd4j.shade.jackson.databind.JsonDeserializer;
import org.nd4j.shade.jackson.databind.JsonNode;
import java.io.IOException;
/**
* Custom deserializer to handle change in format between beta6 (and earlier) and later versions
*
* @author Alex Black
*/
public class BoundingBoxesDeserializer extends JsonDeserializer<INDArray> {
@Override
public INDArray deserialize(JsonParser jp, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
JsonNode node = jp.getCodec().readTree(jp);
if(node.has("dataBuffer")){
//Must be legacy format serialization
JsonNode arr = node.get("dataBuffer");
int rank = node.get("rankField").asInt();
int numElements = node.get("numElements").asInt();
int offset = node.get("offsetField").asInt();
JsonNode shape = node.get("shapeField");
JsonNode stride = node.get("strideField");
int[] shapeArr = new int[rank];
int[] strideArr = new int[rank];
DataBuffer buff = Nd4j.createBuffer(numElements);
for (int i = 0; i < numElements; i++) {
buff.put(i, arr.get(i).asDouble());
}
String ordering = node.get("orderingField").asText();
for (int i = 0; i < rank; i++) {
shapeArr[i] = shape.get(i).asInt();
strideArr[i] = stride.get(i).asInt();
}
return Nd4j.create(buff, shapeArr, strideArr, offset, ordering.charAt(0));
}
//Standard/new format
return new NDArrayTextDeSerializer().deserialize(node);
}
}

View File

@ -34,10 +34,9 @@ import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.learning.regularization.Regularization;
import org.nd4j.linalg.lossfunctions.ILossFunction;
import org.nd4j.linalg.lossfunctions.impl.LossL2;
import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer;
import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize;
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
import org.nd4j.shade.serde.jackson.VectorDeSerializer;
import org.nd4j.shade.serde.jackson.VectorSerializer;
import java.util.Arrays;
import java.util.Collection;
@ -77,8 +76,8 @@ public class Yolo2OutputLayer extends org.deeplearning4j.nn.conf.layers.Layer {
private double lambdaNoObj;
private ILossFunction lossPositionScale;
private ILossFunction lossClassPredictions;
@JsonSerialize(using = VectorSerializer.class)
@JsonDeserialize(using = VectorDeSerializer.class)
@JsonSerialize(using = NDArrayTextSerializer.class)
@JsonDeserialize(using = BoundingBoxesDeserializer.class)
private INDArray boundingBoxes;
private Yolo2OutputLayer() {

View File

@ -44,11 +44,6 @@
<artifactId>deeplearning4j-nlp</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${nd4j.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>

View File

@ -36,6 +36,10 @@ public class NDArrayTextDeSerializer extends JsonDeserializer<INDArray> {
@Override
public INDArray deserialize(JsonParser jp, DeserializationContext deserializationContext) throws IOException {
JsonNode n = jp.getCodec().readTree(jp);
return deserialize(n);
}
public INDArray deserialize(JsonNode n){
//First: check for backward compatilibity (RowVectorSerializer/Deserializer)
if(!n.has("dataType")){

View File

@ -274,12 +274,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-api</artifactId>

View File

@ -76,12 +76,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Shaded version of Jackson -->
<dependency>
<groupId>org.nd4j</groupId>

View File

@ -1,72 +0,0 @@
/*******************************************************************************
* Copyright (c) 2015-2018 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.nd4j.serde.jackson;
import org.junit.BeforeClass;
import org.junit.Test;
import org.nd4j.linalg.BaseNd4jTest;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.factory.Nd4jBackend;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import org.nd4j.shade.jackson.databind.module.SimpleModule;
import org.nd4j.shade.serde.jackson.shaded.NDArrayDeSerializer;
import org.nd4j.shade.serde.jackson.shaded.NDArraySerializer;
import static org.junit.Assert.assertEquals;
/**
* Created by agibsonccc on 6/23/16.
*/
public class NdArraySerializerTest extends BaseNd4jTest {
private static ObjectMapper objectMapper;
public NdArraySerializerTest(Nd4jBackend backend) {
super(backend);
}
@Override
public char ordering() {
return 'c';
}
@BeforeClass
public static void beforeClass() {
objectMapper = objectMapper();
}
@Test
public void testSerde() throws Exception {
String json = objectMapper.writeValueAsString(Nd4j.create(2, 2));
INDArray assertion = Nd4j.create(2, 2);
INDArray test = objectMapper.readValue(json, INDArray.class);
assertEquals(assertion, test);
}
private static ObjectMapper objectMapper() {
ObjectMapper mapper = new ObjectMapper();
SimpleModule nd4j = new SimpleModule("nd4j");
nd4j.addDeserializer(INDArray.class, new NDArrayDeSerializer());
nd4j.addSerializer(INDArray.class, new NDArraySerializer());
mapper.registerModule(nd4j);
return mapper;
}
}

View File

@ -1,72 +0,0 @@
/*******************************************************************************
* Copyright (c) 2015-2018 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
package org.nd4j.serde.jackson;
import org.junit.BeforeClass;
import org.junit.Test;
import org.nd4j.linalg.BaseNd4jTest;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.factory.Nd4jBackend;
import org.nd4j.shade.jackson.databind.ObjectMapper;
import org.nd4j.shade.jackson.databind.module.SimpleModule;
import org.nd4j.shade.serde.jackson.VectorDeSerializer;
import org.nd4j.shade.serde.jackson.VectorSerializer;
import static org.junit.Assert.assertEquals;
/**
* Created by agibsonccc on 6/23/16.
*/
public class VectorSerializeTest extends BaseNd4jTest {
private static ObjectMapper objectMapper;
public VectorSerializeTest(Nd4jBackend backend) {
super(backend);
}
@Override
public char ordering() {
return 'c';
}
@BeforeClass
public static void beforeClass() {
objectMapper = objectMapper();
}
@Test
public void testSerde() throws Exception {
String json = objectMapper.writeValueAsString(Nd4j.create(2, 2));
INDArray assertion = Nd4j.create(2, 2);
INDArray test = objectMapper.readValue(json, INDArray.class);
assertEquals(assertion, test);
}
private static ObjectMapper objectMapper() {
ObjectMapper mapper = new ObjectMapper();
SimpleModule nd4j = new SimpleModule("nd4j");
nd4j.addDeserializer(INDArray.class, new VectorDeSerializer());
nd4j.addSerializer(INDArray.class, new VectorSerializer());
mapper.registerModule(nd4j);
return mapper;
}
}

View File

@ -180,11 +180,6 @@
<artifactId>nd4j-aeron</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-jackson</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-kryo_2.11</artifactId>