From 76aab7517460c95ad4dd5bdef8081f97c2425cbb Mon Sep 17 00:00:00 2001 From: Alex Black Date: Tue, 21 Apr 2020 10:51:26 +1000 Subject: [PATCH] Fix uberjar dependencies (#401) Signed-off-by: Alex Black --- .../nd4j-camel-routes/nd4j-kafka/pom.xml | 95 ---------- .../org/nd4j/kafka/Nd4jKafkaRouteTest.java | 83 --------- nd4j/nd4j-serde/nd4j-gson/pom.xml | 171 ------------------ .../gson/GsonDeserializationUtilsTest.java | 101 ----------- nd4j/nd4j-uberjar/pom.xml | 10 - 5 files changed, 460 deletions(-) delete mode 100644 nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/pom.xml delete mode 100644 nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/src/main/test/java/org/nd4j/kafka/Nd4jKafkaRouteTest.java delete mode 100644 nd4j/nd4j-serde/nd4j-gson/pom.xml delete mode 100644 nd4j/nd4j-serde/nd4j-gson/src/test/java/org/nd4j/serde/gson/GsonDeserializationUtilsTest.java diff --git a/nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/pom.xml b/nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/pom.xml deleted file mode 100644 index 453867c26..000000000 --- a/nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/pom.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - nd4j-camel-routes - org.nd4j - 1.0.0-SNAPSHOT - - 4.0.0 - - nd4j-kafka_2.11 - jar - - nd4j-kafka - https://deeplearning4j.org - - - - 2.11.12 - 2.11 - - - - - org.nd4j - nd4j-api - ${project.version} - - - org.scala-lang - scala-library - ${scala.version} - - - - io.netty - netty - ${netty.version} - - - net.jpountz.lz4 - lz4 - ${lz4.version} - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - org.apache.camel - camel-kafka - ${camel.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - testresources - - - diff --git a/nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/src/main/test/java/org/nd4j/kafka/Nd4jKafkaRouteTest.java b/nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/src/main/test/java/org/nd4j/kafka/Nd4jKafkaRouteTest.java deleted file mode 100644 index 3b40661bb..000000000 --- a/nd4j/nd4j-serde/nd4j-camel-routes/nd4j-kafka/src/main/test/java/org/nd4j/kafka/Nd4jKafkaRouteTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2015-2018 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.nd4j.kafka; - -import org.apache.camel.CamelContext; -import org.apache.camel.impl.DefaultCamelContext; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nd4j.BaseND4JTest; -import org.nd4j.camel.kafka.KafkaConnectionInformation; -import org.nd4j.camel.kafka.Nd4jKafkaConsumer; -import org.nd4j.camel.kafka.Nd4jKafkaProducer; -import org.nd4j.camel.kafka.Nd4jKafkaRoute; -import org.nd4j.linalg.factory.Nd4j; - -import static org.junit.Assert.assertEquals; - -/** - * Created by agibsonccc on 7/19/16. - */ -public class Nd4jKafkaRouteTest extends BaseND4JTest { - private EmbeddedKafkaCluster kafka; - private EmbeddedZookeeper zk; - private CamelContext camelContext; - public final static String TOPIC = "nd4jtest"; - public final static String GROUP_ID = "nd4j"; - private KafkaConnectionInformation connectionInformation; - - - - @Before - public void before() throws Exception { - zk = new EmbeddedZookeeper(TestUtils.getAvailablePort()); - zk.startup(); - kafka = new EmbeddedKafkaCluster(zk.getConnection()); - kafka.startup(); - kafka.createTopics(TOPIC); - camelContext = new DefaultCamelContext(); - camelContext.start(); - connectionInformation = KafkaConnectionInformation.builder().groupId(GROUP_ID).topicName(TOPIC) - .zookeeperHost("localhost").zookeeperPort(zk.getPort()).kafkaBrokerList(kafka.getBrokerList()) - .build(); - camelContext.addRoutes(Nd4jKafkaRoute.builder().kafkaConnectionInformation(connectionInformation).build()); - } - - @After - public void after() throws Exception { - if (kafka != null) - kafka.shutdown(); - if (zk != null) - zk.shutdown(); - if (camelContext != null) - camelContext.stop(); - } - - - @Test - public void testKafkaRoute() throws Exception { - Nd4jKafkaProducer kafkaProducer = Nd4jKafkaProducer.builder().camelContext(camelContext) - .connectionInformation(connectionInformation).build(); - kafkaProducer.publish(Nd4j.create(4)); - Nd4jKafkaConsumer consumer = Nd4jKafkaConsumer.builder().camelContext(camelContext) - .connectionInformation(connectionInformation).build(); - assertEquals(Nd4j.create(4), consumer.receive()); - } - - -} diff --git a/nd4j/nd4j-serde/nd4j-gson/pom.xml b/nd4j/nd4j-serde/nd4j-gson/pom.xml deleted file mode 100644 index 60de01b6e..000000000 --- a/nd4j/nd4j-serde/nd4j-gson/pom.xml +++ /dev/null @@ -1,171 +0,0 @@ - - - - - - nd4j-serde - org.nd4j - 1.0.0-SNAPSHOT - - 4.0.0 - - nd4j-gson - 1.0.0-SNAPSHOT - - - org.nd4j - nd4j-api - ${project.version} - provided - - - - com.google.code.gson - gson - ${gson.version} - provided - - - - junit - junit - test - - - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - testresources - - - - nd4j-tests-cpu - - false - - - - org.nd4j - nd4j-native - ${project.version} - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${env.LD_LIBRARY_PATH}:${user.dir}:${libnd4jhome}/blasbuild/cpu/blas/ - - - src/test/java - - *.java - **/*.java - **/Test*.java - **/*Test.java - **/*TestCase.java - - junit:junit - - org.nd4j.linalg.cpu.nativecpu.CpuBackend - - org.nd4j.linalg.cpu.nativecpu.CpuBackend - - - - -Ddtype=float -Dfile.encoding=UTF-8 -Xmx8g - - - - - - - - nd4j-tests-cuda - - false - - - - org.nd4j - nd4j-cuda-10.2 - ${project.version} - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.maven.surefire - surefire-junit47 - 2.19.1 - - - - - - ${env.LD_LIBRARY_PATH}:${user.dir}:${libnd4jhome}/blasbuild/cuda/blas/ - - - src/test/java - - *.java - **/*.java - **/Test*.java - **/*Test.java - **/*TestCase.java - - junit:junit - - org.nd4j.linalg.jcublas.JCublasBackend - - org.nd4j.linalg.jcublas.JCublasBackend - - - - -Ddtype=float -Xmx6g - - - - - - - - diff --git a/nd4j/nd4j-serde/nd4j-gson/src/test/java/org/nd4j/serde/gson/GsonDeserializationUtilsTest.java b/nd4j/nd4j-serde/nd4j-gson/src/test/java/org/nd4j/serde/gson/GsonDeserializationUtilsTest.java deleted file mode 100644 index a64641fe3..000000000 --- a/nd4j/nd4j-serde/nd4j-gson/src/test/java/org/nd4j/serde/gson/GsonDeserializationUtilsTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2015-2018 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.nd4j.serde.gson; - -import org.junit.Test; -import org.nd4j.BaseND4JTest; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; - -import static org.junit.Assert.assertEquals; - -public class GsonDeserializationUtilsTest extends BaseND4JTest { - @Test - public void deserializeRawJson_PassInInRank3Array_ExpectCorrectDeserialization() { - String serializedRawArray = "[[[1.00, 11.00, 3.00],\n" + "[13.00, 5.00, 15.00],\n" + "[7.00, 17.00, 9.00]]]"; - INDArray expectedArray = buildExpectedArray(1, 1, 3, 3); - - INDArray indArray = GsonDeserializationUtils.deserializeRawJson(serializedRawArray); - - assertEquals(expectedArray, indArray); - } - - @Test - public void deserializeRawJson_ArrayHasOnlyOneRowWithColumns_ExpectCorrectDeserialization() { - String serializedRawArray = "[1.00, 11.00, 3.00]"; - INDArray expectedArray = Nd4j.create(new double[] {1, 11, 3}).castTo(Nd4j.defaultFloatingPointType()); - - INDArray indArray = GsonDeserializationUtils.deserializeRawJson(serializedRawArray); - - assertEquals(expectedArray, indArray); - } - - @Test - public void deserializeRawJson_ArrayIsRankFive_ExpectCorrectDeserialization() { - String serializedRawArray = "[[[[[1.00, 11.00],\n" + " [3.00, 13.00]],\n" + " [[5.00, 15.00],\n" - + " [7.00, 17.00]]],\n" + " [[[9.00, 1.00],\n" + " [11.00, 3.00]],\n" - + " [[13.00, 5.00],\n" + " [15.00, 7.00]]],\n" + " [[[17.00, 9.00],\n" - + " [1.00, 11.00]],\n" + " [[3.00, 13.00],\n" + " [5.00, 15.00]]]],\n" - + " [[[[7.00, 17.00],\n" + " [9.00, 1.00]],\n" + " [[11.00, 3.00],\n" - + " [13.00, 5.00]]],\n" + " [[[15.00, 7.00],\n" + " [17.00, 9.00]],\n" - + " [[1.00, 11.00],\n" + " [3.00, 13.00]]],\n" + " [[[5.00, 15.00],\n" - + " [7.00, 17.00]],\n" + " [[9.00, 1.00],\n" + " [11.00, 3.00]]]],\n" - + " [[[[13.00, 5.00],\n" + " [15.00, 7.00]],\n" + " [[17.00, 9.00],\n" - + " [1.00, 11.00]]],\n" + " [[[3.00, 13.00],\n" + " [5.00, 15.00]],\n" - + " [[7.00, 17.00],\n" + " [9.00, 1.00]]],\n" + " [[[11.00, 3.00],\n" - + " [13.00, 5.00]],\n" + " [[15.00, 7.00],\n" + " [17.00, 9.00]]]]]"; - INDArray expectedArray = buildExpectedArray(8, 3, 3, 2, 2, 2); - - INDArray array = GsonDeserializationUtils.deserializeRawJson(serializedRawArray); - - assertEquals(expectedArray, array); - } - - - @Test - public void testSimpleVector() { - INDArray arr = Nd4j.linspace(1, 4, 4, Nd4j.defaultFloatingPointType()).reshape(4); - INDArray out = GsonDeserializationUtils.deserializeRawJson(arr.toString()); - assertEquals(arr, out); - } - - - - @Test - public void deserializeRawJson_HaveCommaInsideNumbers_ExpectCorrectDeserialization() { - String serializedRawArray = - "[[1.00, 1100.00, 3.00],\n" + "[13.00, 5.00, 15591.00],\n" + "[7000.00, 17.00, 9.00]]"; - INDArray expectedArray = Nd4j.create(new double[] {1, 1100, 3, 13, 5, 15591, 7000, 17, 9}, new int[] {3, 3}).castTo(Nd4j.defaultFloatingPointType()); - - INDArray indArray = GsonDeserializationUtils.deserializeRawJson(serializedRawArray); - - assertEquals(expectedArray, indArray); - } - - private INDArray buildExpectedArray(int numberOfTripletRows, int... shape) { - INDArray expectedArray = Nd4j.create(3 * numberOfTripletRows, 3); - for (int i = 0; i < numberOfTripletRows; i++) { - int index = 3 * i; - expectedArray.putRow(index, Nd4j.create(new double[] {1, 11, 3})); - expectedArray.putRow(index + 1, Nd4j.create(new double[] {13, 5, 15})); - expectedArray.putRow(index + 2, Nd4j.create(new double[] {7, 17, 9})); - } - - return expectedArray.reshape(shape); - } -} diff --git a/nd4j/nd4j-uberjar/pom.xml b/nd4j/nd4j-uberjar/pom.xml index bb0d45b5e..388e48fc5 100644 --- a/nd4j/nd4j-uberjar/pom.xml +++ b/nd4j/nd4j-uberjar/pom.xml @@ -185,16 +185,6 @@ nd4j-kryo_2.11 ${project.version} - - org.nd4j - nd4j-kafka_2.11 - ${project.version} - - - org.nd4j - nd4j-gson - ${project.version} - org.nd4j nd4j-common